Skip to content

Commit 5363406

Browse files
experdotexperdot
committed
refactor: Update configuration value retrieval to use dot notation for improved clarity and consistency across config files
1 parent 28b24bb commit 5363406

File tree

2 files changed

+95
-69
lines changed

2 files changed

+95
-69
lines changed

‎sources/gc-qa-rag-etl/etlapp/common/config.py

Lines changed: 49 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -33,37 +33,50 @@ class VectorDbConfig:
3333

3434

3535
def _get_config_value(key: str, config_raw: dict, saved_config_raw: dict, default: Optional[str] = None) -> str:
36-
"""Get configuration value with priority: saved.json > ENV > .env > JSON."""
37-
# First check saved configuration (highest priority)
38-
keys = key.lower().split('_')
39-
# Skip the 'gc_qa_rag' prefix for JSON lookup
40-
if len(keys) >= 4 and keys[0] == 'gc' and keys[1] == 'qa' and keys[2] == 'rag':
41-
keys = keys[3:]
36+
"""
37+
Get configuration value with priority: saved.json > ENV > .env > JSON.
4238
43-
# Check saved config first
44-
current = saved_config_raw
45-
try:
46-
for k in keys:
47-
current = current[k]
48-
return str(current)
49-
except (KeyError, TypeError):
50-
pass
39+
Key format: GC_QA_RAG.SECTION.FIELD (e.g., GC_QA_RAG.LLM.API_KEY)
40+
- For JSON: uses dot notation to navigate nested structure
41+
- For ENV: converts dots to underscores (GC_QA_RAG_LLM_API_KEY)
42+
"""
43+
44+
def _get_value_from_json(config_dict: dict, dot_key: str) -> Optional[str]:
45+
"""Navigate JSON structure using dot notation."""
46+
if not config_dict:
47+
return None
48+
49+
# Remove GC_QA_RAG prefix and split by dots
50+
if dot_key.startswith("GC_QA_RAG."):
51+
path = dot_key[10:].split('.') # Remove "GC_QA_RAG." prefix
52+
else:
53+
path = dot_key.split('.')
54+
55+
current = config_dict
56+
try:
57+
for key_part in path:
58+
current = current[key_part.lower()]
59+
return str(current)
60+
except (KeyError, TypeError):
61+
return None
62+
63+
# 1. Check saved config first (highest priority)
64+
saved_value = _get_value_from_json(saved_config_raw, key)
65+
if saved_value is not None:
66+
return saved_value
5167

52-
# Then check environment variables
53-
env_value = os.getenv(key)
68+
# 2. Then check environment variables (convert dots to underscores)
69+
env_key = key.replace('.', '_').upper()
70+
env_value = os.getenv(env_key)
5471
if env_value is not None:
5572
return env_value
5673

57-
# Then check nested JSON structure
58-
current = config_raw
59-
try:
60-
for k in keys:
61-
current = current[k]
62-
return str(current)
63-
except (KeyError, TypeError):
64-
pass
74+
# 3. Then check JSON config
75+
json_value = _get_value_from_json(config_raw, key)
76+
if json_value is not None:
77+
return json_value
6578

66-
# Return default if provided
79+
# 4. Return default if provided
6780
if default is not None:
6881
return default
6982

@@ -121,24 +134,24 @@ def from_environment(cls, environment: str) -> "Config":
121134
return cls(
122135
environment=environment,
123136
das=DasConfig(
124-
base_url_page=_get_config_value("GC_QA_RAG_DAS_BASE_URL_PAGE", config_raw, saved_config_raw, ""),
125-
base_url_thread=_get_config_value("GC_QA_RAG_DAS_BASE_URL_THREAD", config_raw, saved_config_raw, ""),
126-
token=_get_config_value("GC_QA_RAG_DAS_TOKEN", config_raw, saved_config_raw, ""),
137+
base_url_page=_get_config_value("GC_QA_RAG.DAS.BASE_URL_PAGE", config_raw, saved_config_raw, ""),
138+
base_url_thread=_get_config_value("GC_QA_RAG.DAS.BASE_URL_THREAD", config_raw, saved_config_raw, ""),
139+
token=_get_config_value("GC_QA_RAG.DAS.TOKEN", config_raw, saved_config_raw, ""),
127140
),
128141
llm=LlmConfig(
129-
api_key=_get_config_value("GC_QA_RAG_LLM_API_KEY", config_raw, saved_config_raw),
130-
api_base=_get_config_value("GC_QA_RAG_LLM_API_BASE", config_raw, saved_config_raw, "https://dashscope.aliyuncs.com/compatible-mode/v1"),
131-
model_name=_get_config_value("GC_QA_RAG_LLM_MODEL_NAME", config_raw, saved_config_raw, "qwen-plus"),
132-
max_rpm=_get_config_int("GC_QA_RAG_LLM_MAX_RPM", config_raw, saved_config_raw, 100),
142+
api_key=_get_config_value("GC_QA_RAG.LLM.API_KEY", config_raw, saved_config_raw),
143+
api_base=_get_config_value("GC_QA_RAG.LLM.API_BASE", config_raw, saved_config_raw, "https://dashscope.aliyuncs.com/compatible-mode/v1"),
144+
model_name=_get_config_value("GC_QA_RAG.LLM.MODEL_NAME", config_raw, saved_config_raw, "qwen-plus"),
145+
max_rpm=_get_config_int("GC_QA_RAG.LLM.MAX_RPM", config_raw, saved_config_raw, 100),
133146
),
134147
embedding=EmbeddingConfig(
135-
api_key=_get_config_value("GC_QA_RAG_EMBEDDING_API_KEY", config_raw, saved_config_raw)
148+
api_key=_get_config_value("GC_QA_RAG.EMBEDDING.API_KEY", config_raw, saved_config_raw)
136149
),
137150
vector_db=VectorDbConfig(
138-
host=_get_config_value("GC_QA_RAG_VECTOR_DB_HOST", config_raw, saved_config_raw, "http://host.docker.internal:6333")
151+
host=_get_config_value("GC_QA_RAG.VECTOR_DB.HOST", config_raw, saved_config_raw, "http://host.docker.internal:6333")
139152
),
140-
root_path=_get_config_value("GC_QA_RAG_ROOT_PATH", config_raw, saved_config_raw, user_cache_dir("gc-qa-rag", ensure_exists=True)),
141-
log_path=_get_config_value("GC_QA_RAG_LOG_PATH", config_raw, saved_config_raw, user_log_dir("gc-qa-rag", ensure_exists=True)),
153+
root_path=_get_config_value("GC_QA_RAG.ROOT_PATH", config_raw, saved_config_raw, user_cache_dir("gc-qa-rag", ensure_exists=True)),
154+
log_path=_get_config_value("GC_QA_RAG.LOG_PATH", config_raw, saved_config_raw, user_log_dir("gc-qa-rag", ensure_exists=True)),
142155
)
143156

144157

‎sources/gc-qa-rag-server/ragapp/common/config.py

Lines changed: 46 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -30,37 +30,50 @@ class DbConfig:
3030

3131

3232
def _get_config_value(key: str, config_raw: dict, saved_config_raw: dict, default: Optional[str] = None) -> str:
33-
"""Get configuration value with priority: saved.json > ENV > .env > JSON."""
34-
# First check saved configuration (highest priority)
35-
keys = key.lower().split('_')
36-
# Skip the 'gc_qa_rag' prefix for JSON lookup
37-
if keys[0] == 'gc' and keys[1] == 'qa' and keys[2] == 'rag':
38-
keys = keys[3:]
33+
"""
34+
Get configuration value with priority: saved.json > ENV > .env > JSON.
3935
40-
# Check saved config first
41-
current = saved_config_raw
42-
try:
43-
for k in keys:
44-
current = current[k]
45-
return str(current)
46-
except (KeyError, TypeError):
47-
pass
36+
Key format: GC_QA_RAG.SECTION.FIELD (e.g., GC_QA_RAG.LLM_DEFAULT.API_KEY)
37+
- For JSON: uses dot notation to navigate nested structure
38+
- For ENV: converts dots to underscores (GC_QA_RAG_LLM_DEFAULT_API_KEY)
39+
"""
4840

49-
# Then check environment variables
50-
env_value = os.getenv(key)
41+
def _get_value_from_json(config_dict: dict, dot_key: str) -> Optional[str]:
42+
"""Navigate JSON structure using dot notation."""
43+
if not config_dict:
44+
return None
45+
46+
# Remove GC_QA_RAG prefix and split by dots
47+
if dot_key.startswith("GC_QA_RAG."):
48+
path = dot_key[10:].split('.') # Remove "GC_QA_RAG." prefix
49+
else:
50+
path = dot_key.split('.')
51+
52+
current = config_dict
53+
try:
54+
for key_part in path:
55+
current = current[key_part.lower()]
56+
return str(current)
57+
except (KeyError, TypeError):
58+
return None
59+
60+
# 1. Check saved config first (highest priority)
61+
saved_value = _get_value_from_json(saved_config_raw, key)
62+
if saved_value is not None:
63+
return saved_value
64+
65+
# 2. Then check environment variables (convert dots to underscores)
66+
env_key = key.replace('.', '_').upper()
67+
env_value = os.getenv(env_key)
5168
if env_value is not None:
5269
return env_value
5370

54-
# Then check nested JSON structure
55-
current = config_raw
56-
try:
57-
for k in keys:
58-
current = current[k]
59-
return str(current)
60-
except (KeyError, TypeError):
61-
pass
71+
# 3. Then check JSON config
72+
json_value = _get_value_from_json(config_raw, key)
73+
if json_value is not None:
74+
return json_value
6275

63-
# Return default if provided
76+
# 4. Return default if provided
6477
if default is not None:
6578
return default
6679

@@ -119,9 +132,9 @@ def _get_llm_config(
119132

120133
# Last resort: use default LLM config from env/JSON
121134
return LlmConfig(
122-
api_key=api_key or _get_config_value("GC_QA_RAG_LLM_DEFAULT_API_KEY", config_raw, saved_config_raw),
123-
api_base=api_base or _get_config_value("GC_QA_RAG_LLM_DEFAULT_API_BASE", config_raw, saved_config_raw, "https://dashscope.aliyuncs.com/compatible-mode/v1"),
124-
model_name=model_name or _get_config_value("GC_QA_RAG_LLM_DEFAULT_MODEL_NAME", config_raw, saved_config_raw, "qwen-plus"),
135+
api_key=api_key or _get_config_value("GC_QA_RAG.LLM_DEFAULT.API_KEY", config_raw, saved_config_raw),
136+
api_base=api_base or _get_config_value("GC_QA_RAG.LLM_DEFAULT.API_BASE", config_raw, saved_config_raw, "https://dashscope.aliyuncs.com/compatible-mode/v1"),
137+
model_name=model_name or _get_config_value("GC_QA_RAG.LLM_DEFAULT.MODEL_NAME", config_raw, saved_config_raw, "qwen-plus"),
125138
)
126139

127140

@@ -177,16 +190,16 @@ def from_environment(cls, environment: str) -> "Config":
177190
llm_query=_get_llm_config(config_raw, saved_config_raw, "llm_query", llm_default),
178191
llm_research=_get_llm_config(config_raw, saved_config_raw, "llm_research", llm_default),
179192
embedding=EmbeddingConfig(
180-
api_key=_get_config_value("GC_QA_RAG_EMBEDDING_API_KEY", config_raw, saved_config_raw)
193+
api_key=_get_config_value("GC_QA_RAG.EMBEDDING.API_KEY", config_raw, saved_config_raw)
181194
),
182195
vector_db=VectorDbConfig(
183-
host=_get_config_value("GC_QA_RAG_VECTOR_DB_HOST", config_raw, saved_config_raw, "http://rag_qdrant_container:6333")
196+
host=_get_config_value("GC_QA_RAG.VECTOR_DB.HOST", config_raw, saved_config_raw, "http://rag_qdrant_container:6333")
184197
),
185198
db=DbConfig(
186-
connection_string=_get_config_value("GC_QA_RAG_DB_CONNECTION_STRING", config_raw, saved_config_raw, "mysql+mysqlconnector://root:12345678@rag_mysql_container:3306/search_db")
199+
connection_string=_get_config_value("GC_QA_RAG.DB.CONNECTION_STRING", config_raw, saved_config_raw, "mysql+mysqlconnector://root:12345678@rag_mysql_container:3306/search_db")
187200
),
188-
log_path=_get_config_value("GC_QA_RAG_LOG_PATH", config_raw, saved_config_raw, user_log_dir("gc-qa-rag-server", ensure_exists=True)),
189-
etl_base_url=_get_config_value("GC_QA_RAG_ETL_BASE_URL", config_raw, saved_config_raw, "http://host.docker.internal:8001"),
201+
log_path=_get_config_value("GC_QA_RAG.LOG_PATH", config_raw, saved_config_raw, user_log_dir("gc-qa-rag-server", ensure_exists=True)),
202+
etl_base_url=_get_config_value("GC_QA_RAG.ETL_BASE_URL", config_raw, saved_config_raw, "http://host.docker.internal:8001"),
190203
)
191204

192205

0 commit comments

Comments
 (0)