Compare commits
No commits in common. "a86e38b40808c2bd37c0f3d1dc1476329227a918" and "8bc760ad8e9b3cf84e505b31df9d20fd22995bec" have entirely different histories.
a86e38b408
...
8bc760ad8e
18
.vscode/settings.json
vendored
18
.vscode/settings.json
vendored
@ -1,18 +0,0 @@
|
|||||||
{
|
|
||||||
"workbench.colorCustomizations": {
|
|
||||||
"titleBar.activeBackground": "#ffb5d5",
|
|
||||||
"titleBar.activeForeground": "#000000",
|
|
||||||
"titleBar.inactiveBackground": "#ffb5d5",
|
|
||||||
"titleBar.inactiveForeground": "#000000",
|
|
||||||
"titleBar.border": "#ffb5d5",
|
|
||||||
"activityBar.background": "#ffb5d5",
|
|
||||||
"activityBar.foreground": "#000000",
|
|
||||||
"statusBar.background": "#ffb5d5",
|
|
||||||
"statusBar.foreground": "#000000",
|
|
||||||
"statusBar.debuggingBackground": "#ffb5d5",
|
|
||||||
"statusBar.debuggingForeground": "#000000",
|
|
||||||
"tab.activeBorder": "#ffb5d5",
|
|
||||||
"iLoveWorkSpaceColors": true,
|
|
||||||
"iLoveWorkSpaceRandom": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
181
LICENSE
181
LICENSE
@ -1,181 +0,0 @@
|
|||||||
Apache License
|
|
||||||
Version 2.0, January 2004
|
|
||||||
http://www.apache.org/licenses/
|
|
||||||
|
|
||||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
||||||
|
|
||||||
1. Definitions.
|
|
||||||
|
|
||||||
"License" shall mean the terms and conditions for use, reproduction,
|
|
||||||
and distribution as defined by Sections 1 through 9 of this document.
|
|
||||||
|
|
||||||
"Licensor" shall mean the copyright owner or entity authorized by
|
|
||||||
the copyright owner that is granting the License.
|
|
||||||
|
|
||||||
"Legal Entity" shall mean the union of the acting entity and all
|
|
||||||
other entities that control, are controlled by, or are under common
|
|
||||||
control with that entity. For the purposes of this definition,
|
|
||||||
"control" means (i) the power, direct or indirect, to cause the
|
|
||||||
direction or management of such entity, whether by contract or
|
|
||||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
||||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
||||||
|
|
||||||
"You" (or "Your") shall mean an individual or Legal Entity
|
|
||||||
exercising permissions granted by this License.
|
|
||||||
|
|
||||||
"Source" form shall mean the preferred form for making modifications,
|
|
||||||
including but not limited to software source code, documentation
|
|
||||||
source, and configuration files.
|
|
||||||
|
|
||||||
"Object" form shall mean any form resulting from mechanical
|
|
||||||
transformation or translation of a Source form, including but
|
|
||||||
not limited to compiled object code, generated documentation,
|
|
||||||
and conversions to other media types.
|
|
||||||
|
|
||||||
"Work" shall mean the work of authorship, whether in Source or
|
|
||||||
Object form, made available under the License, as indicated by a
|
|
||||||
copyright notice that is included in or attached to the Work
|
|
||||||
(an example is provided in the Appendix below).
|
|
||||||
|
|
||||||
"Derivative Works" shall mean any work, whether in Source or Object
|
|
||||||
form, that is based on (or derived from) the Work and for which the
|
|
||||||
copyrighted components of the original Work have been recombined,
|
|
||||||
rearranged, modified, or otherwise modified.
|
|
||||||
|
|
||||||
"Contribution" shall mean any work of authorship, including
|
|
||||||
the original version of the Work and any modifications or additions
|
|
||||||
to that Work or Derivative Works thereof, that is intentionally
|
|
||||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
||||||
or by an individual or Legal Entity authorized to submit on behalf of
|
|
||||||
the copyright owner.
|
|
||||||
|
|
||||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
||||||
on behalf of whom a Contribution has been received by Licensor and
|
|
||||||
subsequently incorporated within the Work.
|
|
||||||
|
|
||||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
||||||
this License, each Contributor hereby grants to You a perpetual,
|
|
||||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
||||||
copyright license to reproduce, prepare Derivative Works of,
|
|
||||||
publicly display, publicly perform, sublicense, and distribute the
|
|
||||||
Work and such Derivative Works in Source or Object form.
|
|
||||||
|
|
||||||
3. Grant of Patent License. Subject to the terms and conditions of
|
|
||||||
this License, each Contributor hereby grants to You a perpetual,
|
|
||||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
||||||
(except as stated in this section) patent license to make, have made,
|
|
||||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
||||||
where such license applies only to those patent claims licensable
|
|
||||||
by such Contributor that are necessarily infringed by their
|
|
||||||
Contribution(s) alone or by combination of their Contribution(s)
|
|
||||||
with the Work to which such Contribution(s) was submitted. If You
|
|
||||||
institute patent litigation against any entity (including a
|
|
||||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
||||||
or a Contribution incorporated within the Work constitutes direct
|
|
||||||
or contributory patent infringement, then any patent licenses
|
|
||||||
granted to You under this License for that Work shall terminate
|
|
||||||
as of the date such litigation is filed.
|
|
||||||
|
|
||||||
4. Redistribution. You may reproduce and distribute copies of the
|
|
||||||
Work or Derivative Works thereof in any medium, with or without
|
|
||||||
modifications, and in Source or Object form, provided that You
|
|
||||||
meet the following conditions:
|
|
||||||
|
|
||||||
(a) You must give any other recipients of the Work or Derivative Works
|
|
||||||
a copy of this License; and
|
|
||||||
|
|
||||||
(b) You must cause any modified files to carry prominent notices stating
|
|
||||||
that You changed the files; and
|
|
||||||
|
|
||||||
(c) You must retain, in the Source form of any Derivative Works that You
|
|
||||||
distribute, all copyright, patent, trademark, and attribution notices
|
|
||||||
from the Source form of the Work, excluding those notices that do not
|
|
||||||
pertain to any part of the Derivative Works; and
|
|
||||||
|
|
||||||
(d) If the Work includes a "NOTICE" text file as part of its
|
|
||||||
distribution, then any Derivative Works that You distribute must
|
|
||||||
include a readable copy of the attribution notices contained
|
|
||||||
within such NOTICE file, excluding those notices that do not
|
|
||||||
pertain to any part of the Derivative Works, in at least one
|
|
||||||
of the following places: within a NOTICE text file distributed
|
|
||||||
as part of the Derivative Works; within the Source form or
|
|
||||||
documentation, if provided along with the Derivative Works; or,
|
|
||||||
within a display generated by the Derivative Works, if and
|
|
||||||
where such third-party notices normally appear. The contents of the
|
|
||||||
NOTICE file are for informational purposes only and do not modify
|
|
||||||
the License. You may add Your own attribution notices within
|
|
||||||
Derivative Works that You distribute, alongside, or as an addendum to
|
|
||||||
the NOTICE text from the Work, provided that such additional
|
|
||||||
attribution notices cannot be construed as modifying the License.
|
|
||||||
|
|
||||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
||||||
any Contribution intentionally submitted for inclusion in the Work
|
|
||||||
by You to the Licensor shall be under the terms and conditions of
|
|
||||||
this License, without any additional terms or conditions.
|
|
||||||
Notwithstanding the above, nothing herein shall supersede or modify
|
|
||||||
the terms of any separate license agreement you may have executed
|
|
||||||
with Licensor regarding such Contributions.
|
|
||||||
|
|
||||||
6. Trademarks. This License does not grant permission to use the trade
|
|
||||||
names, trademarks, service marks, or product names of the Licensor,
|
|
||||||
except as required for reasonable and customary use in describing the
|
|
||||||
origin of the Work and reproducing the content of the NOTICE file.
|
|
||||||
|
|
||||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
||||||
agreed to in writing, Licensor provides the Work (and each
|
|
||||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
implied, including, without limitation, any warranties or conditions
|
|
||||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
||||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
||||||
appropriateness of using or redistributing the Work and assume any
|
|
||||||
risks associated with Your exercise of permissions under this License.
|
|
||||||
|
|
||||||
8. Limitation of Liability. In no event and under no legal theory,
|
|
||||||
whether in tort (including negligence), contract, or otherwise,
|
|
||||||
unless required by applicable law (such as deliberate and grossly
|
|
||||||
negligent acts) or agreed to in writing, shall any Contributor be
|
|
||||||
liable to You for damages, including any direct, indirect, special,
|
|
||||||
incidental, or consequential damages of any character arising as a
|
|
||||||
result of this License or out of the use or inability to use the
|
|
||||||
Work (including but not limited to damages for loss of goodwill,
|
|
||||||
work stoppage, computer failure or malfunction, or any and all
|
|
||||||
other commercial damages or losses), even if such Contributor
|
|
||||||
has been advised of the possibility of such damages.
|
|
||||||
|
|
||||||
9. Accepting Warranty or Additional Liability. While redistributing
|
|
||||||
the Work or Derivative Works thereof, You may choose to offer,
|
|
||||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
||||||
or other liability obligations and/or rights consistent with this
|
|
||||||
License. However, in accepting such obligations, You may act only
|
|
||||||
on Your own behalf and on Your sole responsibility, not on behalf
|
|
||||||
of any other Contributor, and only if You agree to indemnify,
|
|
||||||
defend, and hold each Contributor harmless for any liability
|
|
||||||
incurred by, or claims asserted against, such Contributor by reason
|
|
||||||
of your accepting any such warranty or additional liability.
|
|
||||||
|
|
||||||
END OF TERMS AND CONDITIONS
|
|
||||||
|
|
||||||
APPENDIX: How to apply the Apache License to your work.
|
|
||||||
|
|
||||||
To apply the Apache License to your work, attach the following
|
|
||||||
boilerplate notice, with the fields enclosed by brackets "[]" replaced
|
|
||||||
with your own identifying information. (Don't include the brackets!)
|
|
||||||
The text should be enclosed in the appropriate comment syntax for
|
|
||||||
the file format. We also recommend that a file or class name and
|
|
||||||
description of purpose be included on the same "printed page" as the
|
|
||||||
copyright notice for easier identification within third-party archives.
|
|
||||||
|
|
||||||
Copyright 2025 Yuyao Huang
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
34
README.md
34
README.md
@ -1,35 +1,3 @@
|
|||||||
# lazy_config
|
# lazy_config
|
||||||
|
|
||||||
剥离 detectron2 的 LazyConfig 功能,用于通用任务的项目模板。
|
剥离 detectron2 的 LazyConfig 功能,用于通用任务的项目模板。
|
||||||
|
|
||||||
## How to create your own package based on this package.
|
|
||||||
|
|
||||||
Use the following command to create a new package based on this package:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python init.py your-package-name
|
|
||||||
```
|
|
||||||
|
|
||||||
It will replace the current package name (`lazy_config`) with your own package name in the code base.
|
|
||||||
|
|
||||||
However, you need to modify the author name and email in `pyproject.toml`.
|
|
||||||
|
|
||||||
Then you should use `pip install -e .[dev]` to install the package in editable mode.
|
|
||||||
|
|
||||||
After the project is created, you can delete the `init.py` file and start working on your project.
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
See the example in `projects/lazy_config_demo/main.py` and run it with the following command:
|
|
||||||
|
|
||||||
```
|
|
||||||
PYTHONPATH=. python projects/lazy_config_demo/main.py projects/lazy_config_demo/config.py MODEL.in_features=10
|
|
||||||
```
|
|
||||||
|
|
||||||
## Principles
|
|
||||||
|
|
||||||
- `LazyCall` (or `L`): 这是让你能够以惰性方式定义对象构建的语法糖。
|
|
||||||
|
|
||||||
- `instantiate()` function: 这是将惰性配置(带有 _target_ 的 omegaconf 对象)转换为实际 Python 对象的关键函数。它会处理递归实例化和参数传递。
|
|
||||||
|
|
||||||
- `parse_args_and_configs()` function: 它允许你读取命令行参数和配置文件,并将它们合并成一个 omegaconf 对象。
|
|
||||||
269
init.py
269
init.py
@ -1,269 +0,0 @@
|
|||||||
import argparse
|
|
||||||
import keyword
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import sys
|
|
||||||
|
|
||||||
def main():
|
|
||||||
args = argparse.ArgumentParser()
|
|
||||||
args.add_argument("new_package_name")
|
|
||||||
args.add_argument("--dry-run", action="store_true")
|
|
||||||
args = args.parse_args()
|
|
||||||
|
|
||||||
is_valid_name, reason = is_valid_python_package_name(args.new_package_name)
|
|
||||||
|
|
||||||
assert is_valid_name, reason
|
|
||||||
|
|
||||||
replace_in_multiple_paths([
|
|
||||||
"lazy_config",
|
|
||||||
"projects",
|
|
||||||
"README.md",
|
|
||||||
"pyproject.toml"
|
|
||||||
],
|
|
||||||
old_word="lazy_config",
|
|
||||||
new_word=args.new_package_name,
|
|
||||||
file_extensions=('.py', '.md', '.toml'),
|
|
||||||
dry_run=args.dry_run
|
|
||||||
)
|
|
||||||
|
|
||||||
if not args.dry_run:
|
|
||||||
shutil.move("lazy_config", args.new_package_name)
|
|
||||||
|
|
||||||
|
|
||||||
def is_valid_python_package_name(name: str) -> tuple[bool, str]:
|
|
||||||
"""
|
|
||||||
检查一个字符串是否是合格的 Python 包/模块名。
|
|
||||||
|
|
||||||
这个函数综合考虑了 Python 语法规则和一些常见的最佳实践。
|
|
||||||
|
|
||||||
Args:
|
|
||||||
name (str): 要检查的字符串。
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
tuple[bool, str]: 一个元组,第一个元素是布尔值(True 表示合格,False 表示不合格),
|
|
||||||
第二个元素是不合格的原因或“Valid.”。
|
|
||||||
"""
|
|
||||||
if not name:
|
|
||||||
return False, "包/模块名不能为空。"
|
|
||||||
|
|
||||||
# 将包名按点分割成各个组件
|
|
||||||
segments = name.split('.')
|
|
||||||
|
|
||||||
for segment in segments:
|
|
||||||
if not segment:
|
|
||||||
return False, f"包/模块名 '{name}' 包含空组件 (例如 'a..b' 或开头/结尾是点)。"
|
|
||||||
|
|
||||||
# 规则1: 每个组件都必须是有效的 Python 标识符。
|
|
||||||
# isidentifier() 检查了:
|
|
||||||
# - 以字母或下划线开头 (不能以数字开头)
|
|
||||||
# - 仅包含字母、数字或下划线
|
|
||||||
# - 非空 (虽然上面单独检查了)
|
|
||||||
if not segment.isidentifier():
|
|
||||||
return False, f"组件 '{segment}' 在 '{name}' 中不是一个有效的 Python 标识符。"
|
|
||||||
|
|
||||||
# 规则2: 每个组件都不能是 Python 关键字。
|
|
||||||
if keyword.iskeyword(segment):
|
|
||||||
return False, f"组件 '{segment}' 在 '{name}' 中是一个 Python 关键字。"
|
|
||||||
|
|
||||||
# 规则3 (强烈建议): 避免与内置模块名冲突。
|
|
||||||
# 尽管技术上可以导入,但这会导致混淆和潜在的导入问题。
|
|
||||||
if segment in sys.builtin_module_names:
|
|
||||||
return False, f"组件 '{segment}' 在 '{name}' 中与一个内置的 Python 模块名冲突。"
|
|
||||||
|
|
||||||
# 仅作为PEP 8风格的额外检查,不影响“合格性”,如果需要严格的PEP8,可以将其改为返回False
|
|
||||||
# if not segment.islower() and not segment.startswith('__') and not segment.endswith('__'):
|
|
||||||
# # 允许像 __init__ 这样的特殊名称不是小写
|
|
||||||
# # 对于普通的模块/包名,PEP 8 推荐全小写
|
|
||||||
# # return False, f"组件 '{segment}' 在 '{name}' 中不符合 PEP 8 的全小写命名约定。"
|
|
||||||
# if segment.startswith('_') and not segment == '__init__':
|
|
||||||
# # PEP 8 鼓励避免普通模块/包名的前导下划线
|
|
||||||
# # return False, f"组件 '{segment}' 在 '{name}' 中不符合 PEP 8 的前导下划线约定。"
|
|
||||||
# if segment.endswith('_'):
|
|
||||||
# # PEP 8 鼓励避免普通模块/包名的尾随下划线
|
|
||||||
# # return False, f"组件 '{segment}' 在 '{name}' 中不符合 PEP 8 的尾随下划线约定。"
|
|
||||||
pass
|
|
||||||
|
|
||||||
return True, "Valid."
|
|
||||||
|
|
||||||
|
|
||||||
def replace_in_single_file(file_path: str, old_word: str, new_word: str,
|
|
||||||
encoding: str = 'utf-8', dry_run: bool = False) -> tuple[bool, list[str]]:
|
|
||||||
"""
|
|
||||||
在单个文件中替换特定全字匹配的字符串,并记录每一处替换内容。
|
|
||||||
支持 dry_run 模式。
|
|
||||||
|
|
||||||
Args:
|
|
||||||
file_path (str): 文件的完整路径。
|
|
||||||
old_word (str): 要被替换的全字匹配字符串。
|
|
||||||
new_word (str): 替换成的字符串。
|
|
||||||
encoding (str): 读取和写入文件时使用的编码。
|
|
||||||
dry_run (bool): 如果为 True, 则只记录将要进行的替换,不实际修改文件。
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
tuple[bool, list[str]]:
|
|
||||||
- bool: 是否存在匹配并进行了(或将要进行)替换。
|
|
||||||
- list[str]: 记录了所有替换操作(或模拟操作)的文本日志。
|
|
||||||
每个日志条目包含行号和高亮显示的代码行。
|
|
||||||
"""
|
|
||||||
replacement_logs = []
|
|
||||||
|
|
||||||
try:
|
|
||||||
if not os.path.exists(file_path):
|
|
||||||
replacement_logs.append(f" Error: File not found: {file_path}")
|
|
||||||
return False, replacement_logs
|
|
||||||
|
|
||||||
with open(file_path, 'r', encoding=encoding) as f:
|
|
||||||
content = f.read()
|
|
||||||
|
|
||||||
pattern = r"\b" + re.escape(old_word) + r"\b"
|
|
||||||
matches = list(re.finditer(pattern, content))
|
|
||||||
|
|
||||||
if not matches:
|
|
||||||
return False, [] # 没有匹配,直接返回
|
|
||||||
|
|
||||||
def repl_func(match):
|
|
||||||
match_start = match.start()
|
|
||||||
line_num = content[:match_start].count('\n') + 1
|
|
||||||
|
|
||||||
line_start_idx = content.rfind('\n', 0, match_start) + 1
|
|
||||||
line_end_idx = content.find('\n', match.end())
|
|
||||||
if line_end_idx == -1:
|
|
||||||
line_end_idx = len(content)
|
|
||||||
|
|
||||||
original_line = content[line_start_idx:line_end_idx].strip()
|
|
||||||
|
|
||||||
relative_match_start = match_start - line_start_idx
|
|
||||||
relative_match_end = match.end() - line_start_idx
|
|
||||||
highlighted_line = (
|
|
||||||
original_line[:relative_match_start] +
|
|
||||||
f"**{original_line[relative_match_start:relative_match_end]}**" +
|
|
||||||
original_line[relative_match_end:]
|
|
||||||
)
|
|
||||||
|
|
||||||
replacement_logs.append(f" L{line_num:<4}: '{highlighted_line}' -> '{new_word}'")
|
|
||||||
|
|
||||||
return new_word
|
|
||||||
|
|
||||||
new_content = re.sub(pattern, repl_func, content)
|
|
||||||
|
|
||||||
if not dry_run:
|
|
||||||
with open(file_path, 'w', encoding=encoding) as f:
|
|
||||||
f.write(new_content)
|
|
||||||
|
|
||||||
return True, replacement_logs
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
replacement_logs.append(f" Error processing {file_path}: {e}")
|
|
||||||
return False, replacement_logs
|
|
||||||
|
|
||||||
# --- 新增的辅助函数 (保持不变) ---
|
|
||||||
def _process_a_file(file_path: str, old_word: str, new_word: str,
|
|
||||||
file_extensions: tuple[str, ...], # 变更:现在是元组
|
|
||||||
encoding: str, dry_run: bool, stats: dict):
|
|
||||||
"""
|
|
||||||
辅助函数,封装了对单个文件的处理逻辑并更新统计信息。
|
|
||||||
"""
|
|
||||||
# 检查文件扩展名是否符合要求
|
|
||||||
# 如果 file_extensions 是空元组,表示不限制后缀,总是匹配
|
|
||||||
if file_extensions and not file_path.endswith(file_extensions): # endswith 可以直接接受元组
|
|
||||||
stats['skipped'] += 1
|
|
||||||
return
|
|
||||||
|
|
||||||
found_match, logs = replace_in_single_file(
|
|
||||||
file_path, old_word, new_word, encoding, dry_run
|
|
||||||
)
|
|
||||||
|
|
||||||
if logs:
|
|
||||||
print(f"File: {file_path}")
|
|
||||||
for log_entry in logs:
|
|
||||||
print(log_entry)
|
|
||||||
print("-" * (len(file_path) + 6))
|
|
||||||
|
|
||||||
if found_match:
|
|
||||||
if not dry_run:
|
|
||||||
print(f" ✔ Modified: {file_path}\n")
|
|
||||||
else:
|
|
||||||
print(f" ✔ Will modify: {file_path}\n")
|
|
||||||
stats['modified'] += 1
|
|
||||||
elif not logs and os.path.exists(file_path):
|
|
||||||
stats['skipped'] += 1
|
|
||||||
else:
|
|
||||||
stats['errors'] += 1
|
|
||||||
|
|
||||||
|
|
||||||
# --- 修改后的主协调函数 ---
|
|
||||||
def replace_in_multiple_paths(paths_to_process: list[str], old_word: str, new_word: str,
|
|
||||||
file_extensions: str | tuple[str, ...] = (), # 变更:接受字符串或元组
|
|
||||||
encoding: str = 'utf-8',
|
|
||||||
dry_run: bool = False):
|
|
||||||
"""
|
|
||||||
根据提供的文件和文件夹路径列表,替换所有全字匹配的字符串。
|
|
||||||
支持 dry_run 模式,并允许多种文件后缀过滤。
|
|
||||||
|
|
||||||
Args:
|
|
||||||
paths_to_process (list[str]): 包含要处理的文件和/或文件夹路径的列表。
|
|
||||||
old_word (str): 要被替换的全字匹配字符串。
|
|
||||||
new_word (str): 替换成的字符串。
|
|
||||||
file_extensions (str | tuple[str, ...]):
|
|
||||||
要处理的文件扩展名。可以是单个字符串 (例如 ".py"),
|
|
||||||
也可以是字符串元组 (例如 (".py", ".txt"))。
|
|
||||||
默认 (空元组) 表示处理所有文件扩展名。
|
|
||||||
encoding (str): 读取和写入文件时使用的编码。
|
|
||||||
dry_run (bool): 如果为 True, 则只打印将要进行的替换,不实际修改文件。
|
|
||||||
"""
|
|
||||||
# Normalize file_extensions to always be a tuple, empty if no filter
|
|
||||||
if isinstance(file_extensions, str):
|
|
||||||
if file_extensions == "":
|
|
||||||
normalized_file_extensions = ()
|
|
||||||
else:
|
|
||||||
normalized_file_extensions = (file_extensions,)
|
|
||||||
else:
|
|
||||||
normalized_file_extensions = file_extensions
|
|
||||||
|
|
||||||
action_verb = "Simulating replacement" if dry_run else "Performing replacement"
|
|
||||||
print(f"--- {action_verb} ---")
|
|
||||||
print(f"Searching for full word '{old_word}'")
|
|
||||||
print(f"Replacing with '{new_word}'")
|
|
||||||
print(f"In paths: {paths_to_process}")
|
|
||||||
if normalized_file_extensions:
|
|
||||||
print(f"Filtered by extensions: {','.join(normalized_file_extensions)}")
|
|
||||||
else:
|
|
||||||
print(f"No extension filter (processing all files).")
|
|
||||||
|
|
||||||
if dry_run:
|
|
||||||
print(" (DRY RUN: No files will be modified)")
|
|
||||||
print("-" * 60)
|
|
||||||
|
|
||||||
stats = {'modified': 0, 'skipped': 0, 'errors': 0, 'invalid_paths': 0}
|
|
||||||
|
|
||||||
for path in paths_to_process:
|
|
||||||
if not os.path.exists(path):
|
|
||||||
print(f"‼ Warning: Path does not exist and will be skipped: '{path}'\n")
|
|
||||||
stats['invalid_paths'] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
if os.path.isfile(path):
|
|
||||||
_process_a_file(path, old_word, new_word, normalized_file_extensions, encoding, dry_run, stats)
|
|
||||||
elif os.path.isdir(path):
|
|
||||||
# print(f"Processing directory: '{path}'\n")
|
|
||||||
for root, _, files in os.walk(path):
|
|
||||||
for file_name in files:
|
|
||||||
file_path = os.path.join(root, file_name)
|
|
||||||
_process_a_file(file_path, old_word, new_word, normalized_file_extensions, encoding, dry_run, stats)
|
|
||||||
else:
|
|
||||||
print(f"‼ Warning: Skipping '{path}' (not a file or directory).\n")
|
|
||||||
stats['invalid_paths'] += 1
|
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
|
||||||
print("--- Summary ---")
|
|
||||||
print(f"Total files checked: {stats['modified'] + stats['skipped'] + stats['errors']}")
|
|
||||||
print(f"Files to be modified/modified: {stats['modified']}")
|
|
||||||
print(f"Files skipped (word not found or extension mismatch): {stats['skipped']}")
|
|
||||||
print(f"Files with errors: {stats['errors']}")
|
|
||||||
print(f"Invalid paths provided: {stats['invalid_paths']}")
|
|
||||||
print("=" * 60)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,10 +0,0 @@
|
|||||||
from .instantiate import instantiate
|
|
||||||
from .lazy import LazyCall, LazyConfig
|
|
||||||
from .argparser import parse_args_and_configs
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
"instantiate",
|
|
||||||
"LazyCall",
|
|
||||||
"LazyConfig",
|
|
||||||
"parse_args_and_configs",
|
|
||||||
]
|
|
||||||
@ -1,28 +0,0 @@
|
|||||||
import argparse
|
|
||||||
import os
|
|
||||||
from .lazy import LazyConfig
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args_and_configs(program_description: str = ""):
|
|
||||||
parser = argparse.ArgumentParser(description=program_description)
|
|
||||||
parser.add_argument(
|
|
||||||
"config_file",
|
|
||||||
metavar="FILE",
|
|
||||||
help="path to config file",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"opts",
|
|
||||||
help="Modify config options using the command-line 'KEY VALUE' pairs",
|
|
||||||
default=None,
|
|
||||||
nargs=argparse.REMAINDER,
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if not os.path.exists(args.config_file):
|
|
||||||
raise FileNotFoundError(f"Config file not found: {args.config_file}")
|
|
||||||
cfg = LazyConfig.load(args.config_file)
|
|
||||||
|
|
||||||
if args.opts:
|
|
||||||
cfg.merge_with_dotlist(args.opts)
|
|
||||||
|
|
||||||
return cfg
|
|
||||||
@ -1,101 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
# Original Copyright (c) Facebook, Inc. and its affiliates.
|
|
||||||
# This file has been modified by Yuyao Huang (C) 2025.
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
#
|
|
||||||
# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2)
|
|
||||||
# and is licensed under the Apache License, Version 2.0.
|
|
||||||
# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
import collections.abc as abc
|
|
||||||
import dataclasses
|
|
||||||
import logging
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from lazy_config.utils.registry import _convert_target_to_string, locate
|
|
||||||
|
|
||||||
__all__ = ["dump_dataclass", "instantiate"]
|
|
||||||
|
|
||||||
|
|
||||||
def dump_dataclass(obj: Any):
|
|
||||||
"""
|
|
||||||
Dump a dataclass recursively into a dict that can be later instantiated.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
obj: a dataclass object
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
dict
|
|
||||||
"""
|
|
||||||
assert dataclasses.is_dataclass(obj) and not isinstance(
|
|
||||||
obj, type
|
|
||||||
), "dump_dataclass() requires an instance of a dataclass."
|
|
||||||
ret = {"_target_": _convert_target_to_string(type(obj))}
|
|
||||||
for f in dataclasses.fields(obj):
|
|
||||||
v = getattr(obj, f.name)
|
|
||||||
if dataclasses.is_dataclass(v):
|
|
||||||
v = dump_dataclass(v)
|
|
||||||
if isinstance(v, (list, tuple)):
|
|
||||||
v = [dump_dataclass(x) if dataclasses.is_dataclass(x) else x for x in v]
|
|
||||||
ret[f.name] = v
|
|
||||||
return ret
|
|
||||||
|
|
||||||
|
|
||||||
def instantiate(cfg):
|
|
||||||
"""
|
|
||||||
Recursively instantiate objects defined in dictionaries by
|
|
||||||
"_target_" and arguments.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
cfg: a dict-like object with "_target_" that defines the caller, and
|
|
||||||
other keys that define the arguments
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
object instantiated by cfg
|
|
||||||
"""
|
|
||||||
from omegaconf import ListConfig, DictConfig, OmegaConf
|
|
||||||
|
|
||||||
if isinstance(cfg, ListConfig):
|
|
||||||
lst = [instantiate(x) for x in cfg]
|
|
||||||
return ListConfig(lst, flags={"allow_objects": True})
|
|
||||||
if isinstance(cfg, list):
|
|
||||||
# Specialize for list, because many classes take
|
|
||||||
# list[objects] as arguments, such as ResNet, DatasetMapper
|
|
||||||
return [instantiate(x) for x in cfg]
|
|
||||||
|
|
||||||
# If input is a DictConfig backed by dataclasses (i.e. omegaconf's structured config),
|
|
||||||
# instantiate it to the actual dataclass.
|
|
||||||
if isinstance(cfg, DictConfig) and dataclasses.is_dataclass(cfg._metadata.object_type):
|
|
||||||
return OmegaConf.to_object(cfg)
|
|
||||||
|
|
||||||
if isinstance(cfg, abc.Mapping) and "_target_" in cfg:
|
|
||||||
# conceptually equivalent to hydra.utils.instantiate(cfg) with _convert_=all,
|
|
||||||
# but faster: https://github.com/facebookresearch/hydra/issues/1200
|
|
||||||
cfg = {k: instantiate(v) for k, v in cfg.items()}
|
|
||||||
cls = cfg.pop("_target_")
|
|
||||||
cls = instantiate(cls)
|
|
||||||
|
|
||||||
if isinstance(cls, str):
|
|
||||||
cls_name = cls
|
|
||||||
cls = locate(cls_name)
|
|
||||||
assert cls is not None, cls_name
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
cls_name = cls.__module__ + "." + cls.__qualname__
|
|
||||||
except Exception:
|
|
||||||
# target could be anything, so the above could fail
|
|
||||||
cls_name = str(cls)
|
|
||||||
assert callable(cls), f"_target_ {cls} does not define a callable object"
|
|
||||||
try:
|
|
||||||
return cls(**cfg)
|
|
||||||
except TypeError:
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
logger.error(f"Error when instantiating {cls_name}!")
|
|
||||||
raise
|
|
||||||
return cfg # return as-is if don't know what to do
|
|
||||||
@ -1,449 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
# Original Copyright (c) Facebook, Inc. and its affiliates.
|
|
||||||
# This file has been modified by Yuyao Huang (C) 2025.
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
#
|
|
||||||
# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2)
|
|
||||||
# and is licensed under the Apache License, Version 2.0.
|
|
||||||
# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
import ast
|
|
||||||
import builtins
|
|
||||||
import collections.abc as abc
|
|
||||||
import importlib
|
|
||||||
import inspect
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import uuid
|
|
||||||
from contextlib import contextmanager
|
|
||||||
from copy import deepcopy
|
|
||||||
from dataclasses import is_dataclass
|
|
||||||
from typing import List, Tuple, Union
|
|
||||||
import cloudpickle
|
|
||||||
import yaml
|
|
||||||
from omegaconf import DictConfig, ListConfig, OmegaConf, SCMode
|
|
||||||
|
|
||||||
from lazy_config.utils.file_io import PathManager
|
|
||||||
from lazy_config.utils.registry import _convert_target_to_string
|
|
||||||
|
|
||||||
__all__ = ["LazyCall", "LazyConfig"]
|
|
||||||
|
|
||||||
|
|
||||||
class LazyCall:
|
|
||||||
"""
|
|
||||||
Wrap a callable so that when it's called, the call will not be executed,
|
|
||||||
but returns a dict that describes the call.
|
|
||||||
|
|
||||||
LazyCall object has to be called with only keyword arguments. Positional
|
|
||||||
arguments are not yet supported.
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
::
|
|
||||||
from detectron2.config import instantiate, LazyCall
|
|
||||||
|
|
||||||
layer_cfg = LazyCall(nn.Conv2d)(in_channels=32, out_channels=32)
|
|
||||||
layer_cfg.out_channels = 64 # can edit it afterwards
|
|
||||||
layer = instantiate(layer_cfg)
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, target):
|
|
||||||
if not (callable(target) or isinstance(target, (str, abc.Mapping))):
|
|
||||||
raise TypeError(
|
|
||||||
f"target of LazyCall must be a callable or defines a callable! Got {target}"
|
|
||||||
)
|
|
||||||
self._target = target
|
|
||||||
|
|
||||||
def __call__(self, **kwargs):
|
|
||||||
if is_dataclass(self._target):
|
|
||||||
# omegaconf object cannot hold dataclass type
|
|
||||||
# https://github.com/omry/omegaconf/issues/784
|
|
||||||
target = _convert_target_to_string(self._target)
|
|
||||||
else:
|
|
||||||
target = self._target
|
|
||||||
kwargs["_target_"] = target
|
|
||||||
|
|
||||||
return DictConfig(content=kwargs, flags={"allow_objects": True})
|
|
||||||
|
|
||||||
|
|
||||||
def _visit_dict_config(cfg, func):
|
|
||||||
"""
|
|
||||||
Apply func recursively to all DictConfig in cfg.
|
|
||||||
"""
|
|
||||||
if isinstance(cfg, DictConfig):
|
|
||||||
func(cfg)
|
|
||||||
for v in cfg.values():
|
|
||||||
_visit_dict_config(v, func)
|
|
||||||
elif isinstance(cfg, ListConfig):
|
|
||||||
for v in cfg:
|
|
||||||
_visit_dict_config(v, func)
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_py_syntax(filename):
|
|
||||||
# see also https://github.com/open-mmlab/mmcv/blob/master/mmcv/utils/config.py
|
|
||||||
with PathManager.open(filename, "r") as f:
|
|
||||||
content = f.read()
|
|
||||||
try:
|
|
||||||
ast.parse(content)
|
|
||||||
except SyntaxError as e:
|
|
||||||
raise SyntaxError(f"Config file {filename} has syntax error!") from e
|
|
||||||
|
|
||||||
|
|
||||||
def _cast_to_config(obj):
|
|
||||||
# if given a dict, return DictConfig instead
|
|
||||||
if isinstance(obj, dict):
|
|
||||||
return DictConfig(obj, flags={"allow_objects": True})
|
|
||||||
return obj
|
|
||||||
|
|
||||||
|
|
||||||
_CFG_PACKAGE_NAME = "detectron2._cfg_loader"
|
|
||||||
"""
|
|
||||||
A namespace to put all imported config into.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def _random_package_name(filename):
|
|
||||||
# generate a random package name when loading config files
|
|
||||||
return _CFG_PACKAGE_NAME + str(uuid.uuid4())[:4] + "." + os.path.basename(filename)
|
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def _patch_import():
|
|
||||||
"""
|
|
||||||
Enhance relative import statements in config files, so that they:
|
|
||||||
1. locate files purely based on relative location, regardless of packages.
|
|
||||||
e.g. you can import file without having __init__
|
|
||||||
2. do not cache modules globally; modifications of module states has no side effect
|
|
||||||
3. support other storage system through PathManager, so config files can be in the cloud
|
|
||||||
4. imported dict are turned into omegaconf.DictConfig automatically
|
|
||||||
"""
|
|
||||||
old_import = builtins.__import__
|
|
||||||
|
|
||||||
def find_relative_file(original_file, relative_import_path, level):
|
|
||||||
# NOTE: "from . import x" is not handled. Because then it's unclear
|
|
||||||
# if such import should produce `x` as a python module or DictConfig.
|
|
||||||
# This can be discussed further if needed.
|
|
||||||
relative_import_err = """
|
|
||||||
Relative import of directories is not allowed within config files.
|
|
||||||
Within a config file, relative import can only import other config files.
|
|
||||||
""".replace(
|
|
||||||
"\n", " "
|
|
||||||
)
|
|
||||||
if not len(relative_import_path):
|
|
||||||
raise ImportError(relative_import_err)
|
|
||||||
|
|
||||||
cur_file = os.path.dirname(original_file)
|
|
||||||
for _ in range(level - 1):
|
|
||||||
cur_file = os.path.dirname(cur_file)
|
|
||||||
cur_name = relative_import_path.lstrip(".")
|
|
||||||
for part in cur_name.split("."):
|
|
||||||
cur_file = os.path.join(cur_file, part)
|
|
||||||
if not cur_file.endswith(".py"):
|
|
||||||
cur_file += ".py"
|
|
||||||
if not PathManager.isfile(cur_file):
|
|
||||||
cur_file_no_suffix = cur_file[: -len(".py")]
|
|
||||||
if PathManager.isdir(cur_file_no_suffix):
|
|
||||||
raise ImportError(f"Cannot import from {cur_file_no_suffix}." + relative_import_err)
|
|
||||||
else:
|
|
||||||
raise ImportError(
|
|
||||||
f"Cannot import name {relative_import_path} from "
|
|
||||||
f"{original_file}: {cur_file} does not exist."
|
|
||||||
)
|
|
||||||
return cur_file
|
|
||||||
|
|
||||||
def new_import(name, globals=None, locals=None, fromlist=(), level=0):
|
|
||||||
if (
|
|
||||||
# Only deal with relative imports inside config files
|
|
||||||
level != 0
|
|
||||||
and globals is not None
|
|
||||||
and (globals.get("__package__", "") or "").startswith(_CFG_PACKAGE_NAME)
|
|
||||||
):
|
|
||||||
cur_file = find_relative_file(globals["__file__"], name, level)
|
|
||||||
_validate_py_syntax(cur_file)
|
|
||||||
spec = importlib.machinery.ModuleSpec(
|
|
||||||
_random_package_name(cur_file), None, origin=cur_file
|
|
||||||
)
|
|
||||||
module = importlib.util.module_from_spec(spec)
|
|
||||||
module.__file__ = cur_file
|
|
||||||
with PathManager.open(cur_file) as f:
|
|
||||||
content = f.read()
|
|
||||||
exec(compile(content, cur_file, "exec"), module.__dict__)
|
|
||||||
for name in fromlist: # turn imported dict into DictConfig automatically
|
|
||||||
val = _cast_to_config(module.__dict__[name])
|
|
||||||
module.__dict__[name] = val
|
|
||||||
return module
|
|
||||||
return old_import(name, globals, locals, fromlist=fromlist, level=level)
|
|
||||||
|
|
||||||
builtins.__import__ = new_import
|
|
||||||
yield new_import
|
|
||||||
builtins.__import__ = old_import
|
|
||||||
|
|
||||||
|
|
||||||
class LazyConfig:
|
|
||||||
"""
|
|
||||||
Provide methods to save, load, and overrides an omegaconf config object
|
|
||||||
which may contain definition of lazily-constructed objects.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def load_rel(filename: str, keys: Union[None, str, Tuple[str, ...]] = None):
|
|
||||||
"""
|
|
||||||
Similar to :meth:`load()`, but load path relative to the caller's
|
|
||||||
source file.
|
|
||||||
|
|
||||||
This has the same functionality as a relative import, except that this method
|
|
||||||
accepts filename as a string, so more characters are allowed in the filename.
|
|
||||||
"""
|
|
||||||
caller_frame = inspect.stack()[1]
|
|
||||||
caller_fname = caller_frame[0].f_code.co_filename
|
|
||||||
assert caller_fname != "<string>", "load_rel Unable to find caller"
|
|
||||||
caller_dir = os.path.dirname(caller_fname)
|
|
||||||
filename = os.path.join(caller_dir, filename)
|
|
||||||
return LazyConfig.load(filename, keys)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def load(filename: str, keys: Union[None, str, Tuple[str, ...]] = None):
|
|
||||||
"""
|
|
||||||
Load a config file.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
filename: absolute path or relative path w.r.t. the current working directory
|
|
||||||
keys: keys to load and return. If not given, return all keys
|
|
||||||
(whose values are config objects) in a dict.
|
|
||||||
"""
|
|
||||||
has_keys = keys is not None
|
|
||||||
filename = filename.replace("/./", "/") # redundant
|
|
||||||
if os.path.splitext(filename)[1] not in [".py", ".yaml", ".yml"]:
|
|
||||||
raise ValueError(f"Config file {filename} has to be a python or yaml file.")
|
|
||||||
if filename.endswith(".py"):
|
|
||||||
_validate_py_syntax(filename)
|
|
||||||
|
|
||||||
with _patch_import():
|
|
||||||
# Record the filename
|
|
||||||
module_namespace = {
|
|
||||||
"__file__": filename,
|
|
||||||
"__package__": _random_package_name(filename),
|
|
||||||
}
|
|
||||||
with PathManager.open(filename) as f:
|
|
||||||
content = f.read()
|
|
||||||
# Compile first with filename to:
|
|
||||||
# 1. make filename appears in stacktrace
|
|
||||||
# 2. make load_rel able to find its parent's (possibly remote) location
|
|
||||||
exec(compile(content, filename, "exec"), module_namespace)
|
|
||||||
|
|
||||||
ret = module_namespace
|
|
||||||
else:
|
|
||||||
with PathManager.open(filename) as f:
|
|
||||||
obj = yaml.unsafe_load(f)
|
|
||||||
ret = OmegaConf.create(obj, flags={"allow_objects": True})
|
|
||||||
|
|
||||||
if has_keys:
|
|
||||||
if isinstance(keys, str):
|
|
||||||
return _cast_to_config(ret[keys])
|
|
||||||
else:
|
|
||||||
return tuple(_cast_to_config(ret[a]) for a in keys)
|
|
||||||
else:
|
|
||||||
if filename.endswith(".py"):
|
|
||||||
# when not specified, only load those that are config objects
|
|
||||||
ret = DictConfig(
|
|
||||||
{
|
|
||||||
name: _cast_to_config(value)
|
|
||||||
for name, value in ret.items()
|
|
||||||
if isinstance(value, (DictConfig, ListConfig, dict))
|
|
||||||
and not name.startswith("_")
|
|
||||||
},
|
|
||||||
flags={"allow_objects": True},
|
|
||||||
)
|
|
||||||
return ret
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def save(cfg, filename: str):
|
|
||||||
"""
|
|
||||||
Save a config object to a yaml file.
|
|
||||||
Note that when the config dictionary contains complex objects (e.g. lambda),
|
|
||||||
it can't be saved to yaml. In that case we will print an error and
|
|
||||||
attempt to save to a pkl file instead.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
cfg: an omegaconf config object
|
|
||||||
filename: yaml file name to save the config file
|
|
||||||
"""
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
try:
|
|
||||||
cfg = deepcopy(cfg)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
# if it's deep-copyable, then...
|
|
||||||
def _replace_type_by_name(x):
|
|
||||||
if "_target_" in x and callable(x._target_):
|
|
||||||
try:
|
|
||||||
x._target_ = _convert_target_to_string(x._target_)
|
|
||||||
except AttributeError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# not necessary, but makes yaml looks nicer
|
|
||||||
_visit_dict_config(cfg, _replace_type_by_name)
|
|
||||||
|
|
||||||
save_pkl = False
|
|
||||||
try:
|
|
||||||
dict = OmegaConf.to_container(
|
|
||||||
cfg,
|
|
||||||
# Do not resolve interpolation when saving, i.e. do not turn ${a} into
|
|
||||||
# actual values when saving.
|
|
||||||
resolve=False,
|
|
||||||
# Save structures (dataclasses) in a format that can be instantiated later.
|
|
||||||
# Without this option, the type information of the dataclass will be erased.
|
|
||||||
structured_config_mode=SCMode.INSTANTIATE,
|
|
||||||
)
|
|
||||||
dumped = yaml.dump(dict, default_flow_style=None, allow_unicode=True, width=9999)
|
|
||||||
with PathManager.open(filename, "w") as f:
|
|
||||||
f.write(dumped)
|
|
||||||
|
|
||||||
try:
|
|
||||||
_ = yaml.unsafe_load(dumped) # test that it is loadable
|
|
||||||
except Exception:
|
|
||||||
logger.warning(
|
|
||||||
"The config contains objects that cannot serialize to a valid yaml. "
|
|
||||||
f"{filename} is human-readable but cannot be loaded."
|
|
||||||
)
|
|
||||||
save_pkl = True
|
|
||||||
except Exception:
|
|
||||||
logger.exception("Unable to serialize the config to yaml. Error:")
|
|
||||||
save_pkl = True
|
|
||||||
|
|
||||||
if save_pkl:
|
|
||||||
new_filename = filename + ".pkl"
|
|
||||||
try:
|
|
||||||
# retry by pickle
|
|
||||||
with PathManager.open(new_filename, "wb") as f:
|
|
||||||
cloudpickle.dump(cfg, f)
|
|
||||||
logger.warning(f"Config is saved using cloudpickle at {new_filename}.")
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def apply_overrides(cfg, overrides: List[str]):
|
|
||||||
"""
|
|
||||||
In-place override contents of cfg.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
cfg: an omegaconf config object
|
|
||||||
overrides: list of strings in the format of "a=b" to override configs.
|
|
||||||
See https://hydra.cc/docs/next/advanced/override_grammar/basic/
|
|
||||||
for syntax.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
the cfg object
|
|
||||||
"""
|
|
||||||
|
|
||||||
def safe_update(cfg, key, value):
|
|
||||||
parts = key.split(".")
|
|
||||||
for idx in range(1, len(parts)):
|
|
||||||
prefix = ".".join(parts[:idx])
|
|
||||||
v = OmegaConf.select(cfg, prefix, default=None)
|
|
||||||
if v is None:
|
|
||||||
break
|
|
||||||
if not OmegaConf.is_config(v):
|
|
||||||
raise KeyError(
|
|
||||||
f"Trying to update key {key}, but {prefix} "
|
|
||||||
f"is not a config, but has type {type(v)}."
|
|
||||||
)
|
|
||||||
OmegaConf.update(cfg, key, value, merge=True)
|
|
||||||
|
|
||||||
try:
|
|
||||||
from hydra.core.override_parser.overrides_parser import OverridesParser
|
|
||||||
|
|
||||||
has_hydra = True
|
|
||||||
except ImportError:
|
|
||||||
has_hydra = False
|
|
||||||
|
|
||||||
if has_hydra:
|
|
||||||
parser = OverridesParser.create()
|
|
||||||
overrides = parser.parse_overrides(overrides)
|
|
||||||
for o in overrides:
|
|
||||||
key = o.key_or_group
|
|
||||||
value = o.value()
|
|
||||||
if o.is_delete():
|
|
||||||
# TODO support this
|
|
||||||
raise NotImplementedError("deletion is not yet a supported override")
|
|
||||||
safe_update(cfg, key, value)
|
|
||||||
else:
|
|
||||||
# Fallback. Does not support all the features and error checking like hydra.
|
|
||||||
for o in overrides:
|
|
||||||
key, value = o.split("=")
|
|
||||||
try:
|
|
||||||
value = ast.literal_eval(value)
|
|
||||||
except NameError:
|
|
||||||
pass
|
|
||||||
safe_update(cfg, key, value)
|
|
||||||
return cfg
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def to_py(cfg, prefix: str = "cfg."):
|
|
||||||
"""
|
|
||||||
Try to convert a config object into Python-like psuedo code.
|
|
||||||
|
|
||||||
Note that perfect conversion is not always possible. So the returned
|
|
||||||
results are mainly meant to be human-readable, and not meant to be executed.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
cfg: an omegaconf config object
|
|
||||||
prefix: root name for the resulting code (default: "cfg.")
|
|
||||||
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str of formatted Python code
|
|
||||||
"""
|
|
||||||
import black
|
|
||||||
|
|
||||||
cfg = OmegaConf.to_container(cfg, resolve=True)
|
|
||||||
|
|
||||||
def _to_str(obj, prefix=None, inside_call=False):
|
|
||||||
if prefix is None:
|
|
||||||
prefix = []
|
|
||||||
if isinstance(obj, abc.Mapping) and "_target_" in obj:
|
|
||||||
# Dict representing a function call
|
|
||||||
target = _convert_target_to_string(obj.pop("_target_"))
|
|
||||||
args = []
|
|
||||||
for k, v in sorted(obj.items()):
|
|
||||||
args.append(f"{k}={_to_str(v, inside_call=True)}")
|
|
||||||
args = ", ".join(args)
|
|
||||||
call = f"{target}({args})"
|
|
||||||
return "".join(prefix) + call
|
|
||||||
elif isinstance(obj, abc.Mapping) and not inside_call:
|
|
||||||
# Dict that is not inside a call is a list of top-level config objects that we
|
|
||||||
# render as one object per line with dot separated prefixes
|
|
||||||
key_list = []
|
|
||||||
for k, v in sorted(obj.items()):
|
|
||||||
if isinstance(v, abc.Mapping) and "_target_" not in v:
|
|
||||||
key_list.append(_to_str(v, prefix=prefix + [k + "."]))
|
|
||||||
else:
|
|
||||||
key = "".join(prefix) + k
|
|
||||||
key_list.append(f"{key}={_to_str(v)}")
|
|
||||||
return "\n".join(key_list)
|
|
||||||
elif isinstance(obj, abc.Mapping):
|
|
||||||
# Dict that is inside a call is rendered as a regular dict
|
|
||||||
return (
|
|
||||||
"{"
|
|
||||||
+ ",".join(
|
|
||||||
f"{repr(k)}: {_to_str(v, inside_call=inside_call)}"
|
|
||||||
for k, v in sorted(obj.items())
|
|
||||||
)
|
|
||||||
+ "}"
|
|
||||||
)
|
|
||||||
elif isinstance(obj, list):
|
|
||||||
return "[" + ",".join(_to_str(x, inside_call=inside_call) for x in obj) + "]"
|
|
||||||
else:
|
|
||||||
return repr(obj)
|
|
||||||
|
|
||||||
py_str = _to_str(cfg, prefix=[prefix])
|
|
||||||
try:
|
|
||||||
return black.format_str(py_str, mode=black.Mode())
|
|
||||||
except black.InvalidInput:
|
|
||||||
return py_str
|
|
||||||
@ -1,54 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
# Original Copyright (c) Facebook, Inc. and its affiliates.
|
|
||||||
# This file has been modified by Yuyao Huang (C) 2025.
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
#
|
|
||||||
# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2)
|
|
||||||
# and is licensed under the Apache License, Version 2.0.
|
|
||||||
# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
|
|
||||||
from iopath.common.file_io import HTTPURLHandler, OneDrivePathHandler, PathHandler
|
|
||||||
from iopath.common.file_io import PathManager as PathManagerBase
|
|
||||||
|
|
||||||
__all__ = ["PathManager", "PathHandler"]
|
|
||||||
|
|
||||||
|
|
||||||
PathManager = PathManagerBase()
|
|
||||||
"""
|
|
||||||
This is a detectron2 project-specific PathManager.
|
|
||||||
We try to stay away from global PathManager in fvcore as it
|
|
||||||
introduces potential conflicts among other libraries.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class Detectron2Handler(PathHandler):
|
|
||||||
"""
|
|
||||||
Resolve anything that's hosted under detectron2's namespace.
|
|
||||||
"""
|
|
||||||
|
|
||||||
PREFIX = "detectron2://"
|
|
||||||
S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/"
|
|
||||||
|
|
||||||
def _get_supported_prefixes(self):
|
|
||||||
return [self.PREFIX]
|
|
||||||
|
|
||||||
def _get_local_path(self, path, **kwargs):
|
|
||||||
name = path[len(self.PREFIX) :]
|
|
||||||
return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name, **kwargs)
|
|
||||||
|
|
||||||
def _open(self, path, mode="r", **kwargs):
|
|
||||||
return PathManager.open(
|
|
||||||
self.S3_DETECTRON2_PREFIX + path[len(self.PREFIX) :], mode, **kwargs
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
PathManager.register_handler(HTTPURLHandler())
|
|
||||||
PathManager.register_handler(OneDrivePathHandler())
|
|
||||||
PathManager.register_handler(Detectron2Handler())
|
|
||||||
@ -1,73 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
# Original Copyright (c) Facebook, Inc. and its affiliates.
|
|
||||||
# This file has been modified by Yuyao Huang (C) 2025.
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
#
|
|
||||||
# This file is based on code from Detectron2 (https://github.com/facebookresearch/detectron2)
|
|
||||||
# and is licensed under the Apache License, Version 2.0.
|
|
||||||
# You may obtain a copy of the License at (http://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
import pydoc
|
|
||||||
|
|
||||||
"""
|
|
||||||
`locate` maps a string (typically found in config files) to callable objects.
|
|
||||||
"""
|
|
||||||
|
|
||||||
__all__ = ["locate"]
|
|
||||||
|
|
||||||
|
|
||||||
def _convert_target_to_string(t: Any) -> str:
|
|
||||||
"""
|
|
||||||
Inverse of ``locate()``.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
t: any object with ``__module__`` and ``__qualname__``
|
|
||||||
"""
|
|
||||||
module, qualname = t.__module__, t.__qualname__
|
|
||||||
|
|
||||||
# Compress the path to this object, e.g. ``module.submodule._impl.class``
|
|
||||||
# may become ``module.submodule.class``, if the later also resolves to the same
|
|
||||||
# object. This simplifies the string, and also is less affected by moving the
|
|
||||||
# class implementation.
|
|
||||||
module_parts = module.split(".")
|
|
||||||
for k in range(1, len(module_parts)):
|
|
||||||
prefix = ".".join(module_parts[:k])
|
|
||||||
candidate = f"{prefix}.{qualname}"
|
|
||||||
try:
|
|
||||||
if locate(candidate) is t:
|
|
||||||
return candidate
|
|
||||||
except ImportError:
|
|
||||||
pass
|
|
||||||
return f"{module}.{qualname}"
|
|
||||||
|
|
||||||
|
|
||||||
def locate(name: str) -> Any:
|
|
||||||
"""
|
|
||||||
Locate and return an object ``x`` using an input string ``{x.__module__}.{x.__qualname__}``,
|
|
||||||
such as "module.submodule.class_name".
|
|
||||||
|
|
||||||
Raise Exception if it cannot be found.
|
|
||||||
"""
|
|
||||||
obj = pydoc.locate(name)
|
|
||||||
|
|
||||||
# Some cases (e.g. torch.optim.sgd.SGD) not handled correctly
|
|
||||||
# by pydoc.locate. Try a private function from hydra.
|
|
||||||
if obj is None:
|
|
||||||
try:
|
|
||||||
# from hydra.utils import get_method - will print many errors
|
|
||||||
from hydra.utils import _locate
|
|
||||||
except ImportError as e:
|
|
||||||
raise ImportError(
|
|
||||||
f"Cannot dynamically locate object {name}!") from e
|
|
||||||
else:
|
|
||||||
obj = _locate(name) # it raises if fails
|
|
||||||
|
|
||||||
return obj
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
from lazy_config.utils.config import LazyCall as L
|
|
||||||
from projects.lazy_config_demo.model import SimpleModule
|
|
||||||
|
|
||||||
|
|
||||||
MODEL = L(SimpleModule)(
|
|
||||||
in_features=128,
|
|
||||||
out_features=256,
|
|
||||||
activation="sigmoid"
|
|
||||||
)
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
from lazy_config.utils.config import parse_args_and_configs, instantiate
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
cfg = parse_args_and_configs()
|
|
||||||
model = instantiate(cfg.MODEL)
|
|
||||||
print(model)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
class SimpleModule:
|
|
||||||
|
|
||||||
def __init__(self,
|
|
||||||
in_features: int,
|
|
||||||
out_features: int,
|
|
||||||
activation: str = "relu"):
|
|
||||||
self.in_features = in_features
|
|
||||||
self.out_features = out_features
|
|
||||||
self.activation = activation
|
|
||||||
print(
|
|
||||||
f"SimpleModule created: {in_features} -> {out_features}, activation: {activation}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@ -1,38 +0,0 @@
|
|||||||
[build-system]
|
|
||||||
requires = ["setuptools>=65.5.0", "wheel"]
|
|
||||||
build-backend = "setuptools.build_meta"
|
|
||||||
|
|
||||||
[project]
|
|
||||||
name = "lazy_config" # pip install
|
|
||||||
version = "0.1.0"
|
|
||||||
description = "..."
|
|
||||||
readme = "README.md" # 可选, 如果你有 README.md 文件
|
|
||||||
requires-python = ">=3.10"
|
|
||||||
license = {text = "Apache-2.0"} # 或者你的许可证
|
|
||||||
authors = [
|
|
||||||
{name = "Yuyao Huang (Sam)", email = "huangyuyao@outlook.com"},
|
|
||||||
]
|
|
||||||
dependencies = [
|
|
||||||
"cloudpickle",
|
|
||||||
"omegaconf",
|
|
||||||
"iopath",
|
|
||||||
"hydra-core",
|
|
||||||
]
|
|
||||||
|
|
||||||
[tool.setuptools]
|
|
||||||
py-modules = ["lazy_config"]
|
|
||||||
|
|
||||||
# 包含非 Python 文件
|
|
||||||
# [tool.setuptools.package-data]
|
|
||||||
# "package.path" = ["*.json"]
|
|
||||||
|
|
||||||
# 命令行程序入口
|
|
||||||
# [project.scripts]
|
|
||||||
# script-name = "module.path:function"
|
|
||||||
|
|
||||||
[project.optional-dependencies]
|
|
||||||
dev = [
|
|
||||||
"pytest",
|
|
||||||
"pytest-cov",
|
|
||||||
"pytest-mock",
|
|
||||||
]
|
|
||||||
Loading…
x
Reference in New Issue
Block a user