1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532#!/usr/bin/env python3
# tools/process_config.py
#
# SPDX-License-Identifier: Apache-2.0
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership. The
# ASF licenses this file to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import json
import os
import re
import shutil
import sys
from collections import OrderedDict
from pathlib import Path
def parse_config_line(line):
"""
Parse a configuration line and return the key and value.
Args:
line (str): A line from a configuration file
Returns:
tuple: (key, value) if the line contains a configuration, (None, None) otherwise
Handles two formats:
1. "# CONFIG_XXX is not set" -> returns (CONFIG_XXX, 'n')
2. "CONFIG_XXX=value" -> returns (CONFIG_XXX, value)
"""
line = line.strip()
if not line:
return None, None
# Handle "# CONFIG_XXX is not set" format
if line.startswith("# ") and line.endswith(" is not set"):
config_name = line.split()[1]
return config_name, "n"
# Handle "CONFIG_XXX=value" format
if "=" in line:
key, value = line.split("=", 1)
return key, value
return None, None
def opposite(value):
if value == "n":
return "y"
else:
return "n"
def expand_file(input_path, include_paths, processed=None, tree_node=None):
"""
Recursively expand a configuration file with #include directives.
Args:
input_path (str): Path to the input configuration file
include_paths (list): List of directories to search for included files
processed (set, optional): Set of already processed files to avoid circular includes
tree_node (dict, optional): Node in the configuration tree being built
Returns:
tuple: (list of expanded lines, tree structure node)
This function:
1. Reads the input file line by line
2. Processes #include directives by recursively expanding included files
3. Parses configuration lines to build a configuration dictionary
4. Builds a tree structure representing the file inclusion hierarchy
5. Returns the expanded content and the tree structure
"""
if processed is None:
processed = set()
if tree_node is None:
tree_node = {
"file": str(input_path),
"includes": [],
"configs": OrderedDict(),
"include_lines": [],
"raw_content": [], # Store original content for postprocessing
}
input_path = Path(input_path).resolve()
if input_path in processed:
return [], tree_node
processed.add(input_path)
expanded_lines = []
current_configs = OrderedDict()
with input_path.open("r", encoding="utf-8") as f:
lines = f.readlines()
# Save original content for postprocessing
tree_node["raw_content"] = [line.rstrip("\n") for line in lines]
for line in lines:
line_strip = line.strip()
match = re.match(r"#include\s*[<\"]([^\">]+)[\">]", line_strip)
if match:
include_file = match.group(1)
found = False
# Record original include line for postprocessing
tree_node["include_lines"].append(line.rstrip("\n"))
# Check current directory first
direct_path = input_path.parent / include_file
if direct_path.exists():
include_node = {
"file": str(direct_path),
"includes": [],
"configs": OrderedDict(),
"include_lines": [],
"raw_content": [],
}
tree_node["includes"].append(include_node)
# Recursively expand the included file
included_lines, include_node = expand_file(
direct_path, include_paths, processed, include_node
)
expanded_lines.extend(included_lines)
# Merge configurations (later configurations override earlier ones)
for key, value in include_node["configs"].items():
current_configs[key] = value
tree_node["configs"][key] = value
found = True
else:
# Check include paths
for path in include_paths:
candidate = Path(path) / include_file
if candidate.exists():
include_node = {
"file": str(candidate),
"includes": [],
"configs": OrderedDict(),
"include_lines": [],
"raw_content": [],
}
tree_node["includes"].append(include_node)
# Recursively expand the included file
included_lines, include_node = expand_file(
candidate, include_paths, processed, include_node
)
expanded_lines.extend(included_lines)
# Merge configurations
for key, value in include_node["configs"].items():
current_configs[key] = value
tree_node["configs"][key] = value
found = True
break
if not found:
print(
f'ERROR: Cannot find "{include_file}" from {input_path}',
file=sys.stderr,
)
sys.exit(1)
else:
# Parse configuration line
key, value = parse_config_line(line)
if key is not None:
current_configs[key] = value
tree_node["configs"][key] = value
expanded_lines.append(line)
expanded_lines.append("\n") # Maintain separation between files
return expanded_lines, tree_node
def preprocess(output_path, input_path, include_paths, tree_output_path=None):
"""
Process a configuration file with #include directives.
Args:
output_path (str): Path to write the expanded configuration
input_path (str): Path to the input configuration file
include_paths (list): List of directories to search for included files
tree_output_path (str, optional): Path to write the tree structure
This function:
1. Expands the input file by processing #include directives
2. Writes the expanded configuration to output_path
3. Optionally writes the tree structure to tree_output_path for postprocessing
"""
lines, tree = expand_file(input_path, include_paths)
# Write expanded configuration
with open(output_path, "w", encoding="utf-8") as out:
out.writelines(lines)
# Write tree structure if requested
if tree_output_path and tree["includes"]:
with open(tree_output_path, "w", encoding="utf-8") as f:
json.dump(tree, f, indent=2, ensure_ascii=False)
def get_all_included_configs(tree):
"""
Extract all configuration options from included files.
Args:
tree (dict): The configuration tree structure
Returns:
OrderedDict: Dictionary of configuration options from all included files
This function recursively traverses the tree to collect all configurations
from files included via #include directives.
"""
included_configs = OrderedDict()
def collect_configs(node):
for include in node.get("includes", []):
collect_configs(include)
for key, value in node.get("configs", {}).items():
included_configs[key] = value
# Collect configurations from included files only (not the main file)
for include in tree.get("includes", []):
collect_configs(include)
return included_configs
def get_main_configs(tree):
"""
Extract configuration options from the main file (excluding #include directives).
Args:
tree (dict): The configuration tree structure
Returns:
OrderedDict: Dictionary of configuration options from the main file
"""
main_configs = OrderedDict()
for line in tree["raw_content"]:
key, value = parse_config_line(line)
if key is not None:
main_configs[key] = value
return main_configs
def get_current_configs(config_path):
"""
Parse the current full configuration file.
Args:
config_path (str): Path to the current configuration file
Returns:
OrderedDict: Dictionary of configuration options from the current file
"""
configs = OrderedDict()
with open(config_path, "r", encoding="utf-8") as f:
for line in f:
key, value = parse_config_line(line)
if key is not None:
configs[key] = value
return configs
def postprocess_inner(tree_path, added, changed, removed, output_path):
"""
Postprocess configuration changes to generate a defconfig with #include directives.
This function takes the specific changes (added, changed, removed) calculated
by postprocess and applies them to the original defconfig structure
represented by the tree, producing a new defconfig file.
Args:
tree_path (str): Path to the config_tree.json generated during preprocessing of the ORIGINAL defconfig.
added (dict): {key: value} - Configurations added by the user.
changed (dict): {key: (old_value, new_value)} - Configurations changed by the user.
removed (dict): {key: old_value} - Configurations removed by the user.
output_path (str): Path where the new defconfig should be written.
"""
# 1. Load the original tree structure (this represents the structure of the ORIGINAL defconfig)
with open(tree_path, "r", encoding="utf-8") as f:
original_tree = json.load(f, object_pairs_hook=OrderedDict)
# 2. Get the original configuration sets from the tree
original_included_configs = get_all_included_configs(original_tree)
original_main_configs = get_main_configs(original_tree)
# 3. Dictionary to store the final configurations that will go into the main defconfig file
final_main_configs = OrderedDict()
# --- Logic to determine final content of the main defconfig file ---
# a. Handle configurations that were originally in included files
# We only place them in the main defconfig if they were explicitly added/changed/removed.
# If untouched, they remain in their included files implicitly.
for key in original_included_configs:
if key in added:
# User added/changed a config that was originally in an included file.
# It must now be explicitly set in the main defconfig to override the included value.
final_main_configs[key] = added[key]
elif key in changed:
# User changed a config that was originally in an included file.
final_main_configs[key] = changed[key][1] # Use the new value
elif key in removed:
# User removed a config that was originally in an included file.
# To "remove" it, we explicitly set it to opposite orig value in the main defconfig.
# This overrides the value from the included file.
final_main_configs[key] = opposite(removed[key])
# b. Handle configurations that were originally in the main file
# They should generally stay represented in the main file output.
for key in original_main_configs:
if key in added:
# User added/changed a config that was already in the main file.
final_main_configs[key] = added[key]
elif key in changed:
# User changed a config that was in the main file.
final_main_configs[key] = changed[key][1] # Use the new value
elif key in removed:
# User removed a config that was in the main file.
# Explicitly set to opposite orig value to override its previous state.
final_main_configs[key] = opposite(removed[key])
else:
# Config was in the original main file and user did NOT touch it.
# According to the new logic, we should PRESERVE these in the output main defconfig
# to maintain the structure and non-default values from the original main file.
# This prevents the output from becoming sparse if the user only made minor changes.
final_main_configs[key] = original_main_configs[key]
# c. Handle configurations that are entirely new (not present in original main or included)
# These must go into the main defconfig file.
for key, value in added.items():
if key not in original_main_configs and key not in original_included_configs:
final_main_configs[key] = value
# 4. Write the final output defconfig file
with open(output_path, "w", encoding="utf-8") as f:
# Write the original #include directives to preserve the structure
for include_line in original_tree.get("include_lines", []):
f.write(include_line + "\n")
# Add a newline for separation if there were includes
if original_tree.get("include_lines"):
f.write("\n")
# Write the final configurations for the main file in sorted order
final_write_list = []
for key, value in final_main_configs.items():
if value == "n":
final_write_list.append(f"# {key} is not set\n")
else:
final_write_list.append(f"{key}={value}\n")
# Sort configurations for consistent and readable output
final_write_list.sort()
for write_line in final_write_list:
f.write(write_line)
def get_config_diff(old_config, new_config):
"""
Compare two config dictionaries and return the differences.
Args:
old_config (dict): The original configuration.
new_config (dict): The modified configuration.
Returns:
tuple: (added, changed, removed)
added (dict): Items in new_config but not in old_config.
changed (dict): Items with different values. {key: (old_value, new_value)}.
removed (dict): Items in old_config but not in new_config.
"""
added = {}
changed = {}
removed = {}
# Find added and changed items
for key, new_value in new_config.items():
if key not in old_config:
added[key] = new_value
elif old_config[key] != new_value:
changed[key] = (old_config[key], new_value) # (old_value, new_value)
# Find removed items
for key in old_config:
if key not in new_config:
removed[key] = old_config[key]
return added, changed, removed
def load_config_file(filepath):
"""
Load a .config or defconfig file into an OrderedDict.
"""
config = OrderedDict()
try:
with open(filepath, "r") as f:
for line in f:
key, value = parse_config_line(line)
if key is not None:
config[key] = value
except FileNotFoundError:
print(
f"Warning: Config file {filepath} not found. Treating as empty.",
file=sys.stderr,
)
return config
def postprocess(
tree_path, original_defconfig_path, modified_defconfig_path, output_defconfig_path
):
"""
An improved postprocess that compares defconfig files before and after modification.
This function addresses the issue where Kconfig's savedefconfig omits default values,
making it hard to distinguish user deletions from optimizations.
Args:
tree_path (str): Path to the config_tree.json generated during preprocessing.
original_defconfig_path (str): Path to the defconfig file BEFORE user modification.
modified_defconfig_path (str): Path to the defconfig file AFTER user modification.
output_defconfig_path (str): Path where the updated defconfig should be written.
"""
# 1. Load the defconfig files
defconfig_original = load_config_file(original_defconfig_path)
defconfig_modified = load_config_file(modified_defconfig_path)
# 2. Compare the defconfig files to find the actual user changes
added, changed, removed = get_config_diff(defconfig_original, defconfig_modified)
# 3. Use the new postprocess_inner function to generate the final defconfig
# Pass the calculated differences (added, changed, removed) and the original tree.
postprocess_inner(tree_path, added, changed, removed, output_defconfig_path)
if __name__ == "__main__":
if len(sys.argv) < 3:
print(
"Usage: process_config.py <mode> [options]",
file=sys.stderr,
)
print("Modes:", file=sys.stderr)
print(
" preprocess <output_file> <input_file> [include_paths...] [--tree <tree_file>]",
file=sys.stderr,
)
print(
" postprocess <tree_file> <original_defconfig> <modified_defconfig> <output_defconfig>",
file=sys.stderr,
)
sys.exit(1)
mode = sys.argv[1]
if mode == "preprocess":
if len(sys.argv) < 4:
print(
"Usage: preprocess <output_file> <input_file> [include_paths...] [--tree <tree_file>]",
file=sys.stderr,
)
sys.exit(1)
output_file = Path(sys.argv[2])
input_file = sys.argv[3]
include_dirs = []
tree_file = None
# Parse arguments
i = 4
while i < len(sys.argv):
if sys.argv[i] == "--tree" and i + 1 < len(sys.argv):
tree_file = sys.argv[i + 1]
i += 2
else:
include_dirs.append(sys.argv[i])
i += 1
if output_file.exists():
output_file.unlink()
preprocess(output_file, input_file, include_dirs, tree_file)
elif mode == "postprocess":
if len(sys.argv) < 6:
print(
"Usage: postprocess <tree_file> <original_defconfig> <modified_defconfig> <output_defconfig>",
file=sys.stderr,
)
sys.exit(1)
tree_file = sys.argv[2]
original_defconfig = sys.argv[3]
modified_defconfig = sys.argv[4]
output_defconfig = sys.argv[5]
if Path(tree_file).is_file():
post_defconfig = output_defconfig + "tmp"
postprocess(
tree_file, original_defconfig, modified_defconfig, post_defconfig
)
shutil.copy2(post_defconfig, output_defconfig)
os.remove(post_defconfig)
else:
shutil.copy2(modified_defconfig, output_defconfig)
else:
print(f"Unknown mode: {mode}", file=sys.stderr)
sys.exit(1)