1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172#!/usr/bin/python
import argparse
import asyncio
import colorlog
from collections import defaultdict
import logging
import pyalpm
import signal
import sys
parser = argparse.ArgumentParser(description='A simple rebuild scheduler')
parser.add_argument('-d', '--db', nargs='*', default=["core", "extra", "community", "multilib"],
help='Pacman sync db to consider. Defaulting to all stable dbs.')
parser.add_argument('-m', '--mock', nargs='?', default=None,
help='Mock build, do not actually run builds. set to a threshold (0-1, chance to succeed).')
parser.add_argument('-r', '--retry', nargs='?', default=5,
help='Maximum retry on failure, set to 0 to disable retry. Default: 5')
parser.add_argument('-l', '--log', nargs='?', default="dorebuild.log",
help="Log filename. Default: dorebuild.log")
parser.add_argument('-c', '--command', nargs='?', default="rebuild-one",
help='Build command to use. Default: rebuild-one')
parser.add_argument('-j', '--jobs', nargs='?', default="10",
help='Concurrent jobs limit. Default: 10')
parser.add_argument('comment', nargs=1, help='Commit message')
parser.add_argument('package', nargs='+', help='Packages to rebuild')
args = parser.parse_args()
logger = colorlog.getLogger()
logger.setLevel(colorlog.DEBUG)
formatter = colorlog.ColoredFormatter()
handler = colorlog.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
if args.log:
fh = logging.FileHandler(args.log)
fh.setFormatter(formatter)
logger.addHandler(fh)
package_db = defaultdict(dict)
handle = pyalpm.Handle(".", "/var/lib/pacman")
for db in args.db:
db_handle = handle.register_syncdb(db, 0)
for package in db_handle.search(""):
for field in ("depends", "makedepends", "checkdepends", "arch"):
package_db[package.name][field] = getattr(package, field)
for pkg in package_db:
package_db[pkg]["depends"] = {dep.split("=")[0].split(">")[0].split("<")[0] for dep in package_db[pkg]["depends"]}
package_db[pkg]["makedepends"] = {dep.split("=")[0].split(">")[0].split("<")[0] for dep in package_db[pkg]["makedepends"]}
package_db[pkg]["checkdepends"] = {dep.split("=")[0].split(">")[0].split("<")[0] for dep in package_db[pkg]["checkdepends"]}
async def main():
ongoing_pkgs = set()
loop = asyncio.get_event_loop()
main.broken = False
main.next_try = []
async def status_report():
while True:
await asyncio.sleep(60)
logger.debug(f"ongoing_pkgs: {ongoing_pkgs}")
loop.create_task(status_report())
async def run(*cmd):
_BUFFER_SIZE = 2 * 2 ** 20 # 2 MiB buffer!
if args.mock:
import random
await asyncio.sleep(random.random() * 3)
return random.random() > float(args.mock)
try:
proc = await asyncio.create_subprocess_exec(
*cmd,
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,
limit=_BUFFER_SIZE)
while True:
try:
line = await proc.stdout.readline()
except ValueError:
# Underlying asyncio.exceptions.LimitOverrunError
line = await proc.stdout.read(_BUFFER_SIZE)
if line:
logger.debug(f'{cmd[-1]}: ' + line.decode("utf-8", "ignore").rstrip('\n'))
else:
break
return await proc.wait()
finally:
try:
proc.terminate()
except:
pass
async def rebuild_single(pkg):
if not main.broken:
logger.debug(f"Starting rebuild of {pkg}")
retry_count = 0
while ret := await run(args.command, args.comment[0], pkg):
if int(args.retry) and retry_count < int(args.retry):
logger.error(f"Failed rebuild of {pkg}! (ret: {ret}) retry: {retry_count}")
retry_count += 1
else:
logger.error(f"Failed rebuild of {pkg}! (ret: {ret})")
if main.broken is False:
main.broken = [pkg]
else:
main.broken.append(pkg)
main.next_try.append(pkg)
break
else:
if retry_count:
logger.info(f"Finished rebuild of {pkg} retried: {retry_count}")
else:
logger.info(f"Finished rebuild of {pkg}")
else:
main.next_try.append(pkg)
ongoing_pkgs.remove(pkg)
def handler(*args, **kwargs):
logger.error("Received SIGINT, exiting...")
logger.warning("Ongoing packages: " + ", ".join(ongoing_pkgs))
sys.exit(1)
signal.signal(signal.SIGINT, handler)
for pkg in args.package:
pending = True
while pending:
for _pkg in ongoing_pkgs:
pkg_ = pkg
if pkg_.endswith(":nocheck"):
pkg_ = pkg_[:-8]
if _pkg.endswith(":nocheck"):
_pkg = _pkg[:-8]
# New package, no enough info to check dependencies
if pkg_ not in package_db:
break
if _pkg not in package_db:
break
if _pkg in package_db[pkg_]["depends"] | package_db[pkg_]["makedepends"] | package_db[pkg_]["checkdepends"] or \
pkg_ in package_db[_pkg]["depends"] | package_db[_pkg]["makedepends"] | package_db[_pkg]["checkdepends"] or \
_pkg == pkg_:
break
else:
pending = False
while len(ongoing_pkgs) > int(args.jobs):
await asyncio.sleep(0.1)
ongoing_pkgs.add(pkg)
loop.create_task(rebuild_single(pkg))
await asyncio.sleep(0.01)
while ongoing_pkgs:
await asyncio.sleep(0.1)
if main.broken:
logger.warning("Broken: " + " ".join(main.broken))
logger.warning("Next try: " + " ".join(main.next_try))
if __name__ == "__main__":
asyncio.run(main())