1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
|
#!/usr/bin/python3
"""Build cephadm from one or more files into a standalone executable.
"""
# TODO: If cephadm is being built and packaged within a format such as RPM
# do we have to do anything special wrt passing in the version
# of python to build with? Even with the intermediate cmake layer?
import argparse
import compileall
import logging
import os
import pathlib
import shutil
import subprocess
import tempfile
import sys
HAS_ZIPAPP = False
try:
import zipapp
HAS_ZIPAPP = True
except ImportError:
pass
log = logging.getLogger(__name__)
_VALID_VERS_VARS = [
"CEPH_GIT_VER",
"CEPH_GIT_NICE_VER",
"CEPH_RELEASE",
"CEPH_RELEASE_NAME",
"CEPH_RELEASE_TYPE",
]
def _reexec(python):
"""Switch to the selected version of python by exec'ing into the desired
python path.
Sets the _BUILD_PYTHON_SET env variable as a sentinel to indicate exec has
been performed.
"""
env = os.environ.copy()
env["_BUILD_PYTHON_SET"] = python
os.execvpe(python, [python, __file__] + sys.argv[1:], env)
def _did_rexec():
"""Returns true if the process has already exec'ed into the desired python
version.
"""
return bool(os.environ.get("_BUILD_PYTHON_SET", ""))
def _build(dest, src, versioning_vars=None):
"""Build the binary."""
os.chdir(src)
tempdir = pathlib.Path(tempfile.mkdtemp(suffix=".cephadm.build"))
log.debug("working in %s", tempdir)
try:
if os.path.isfile("requirements.txt"):
_install_deps(tempdir)
log.info("Copying contents")
# TODO: currently the only file relevant to a compiled cephadm is the
# cephadm.py file. Once cephadm is broken up into multiple py files
# (and possibly other libs from python-common, etc) we'll want some
# sort organized structure to track what gets copied into the
# dir to be zipped. For now we just have a simple call to copy
# (and rename) the one file we care about.
shutil.copy("cephadm.py", tempdir / "__main__.py")
if versioning_vars:
generate_version_file(versioning_vars, tempdir / "_version.py")
_compile(dest, tempdir)
finally:
shutil.rmtree(tempdir)
def _compile(dest, tempdir):
"""Compile the zipapp."""
log.info("Byte-compiling py to pyc")
compileall.compile_dir(
tempdir,
maxlevels=16,
legacy=True,
quiet=1,
workers=0,
)
# TODO we could explicitly pass a python version here
log.info("Constructing the zipapp file")
try:
zipapp.create_archive(
source=tempdir,
target=dest,
interpreter=sys.executable,
compressed=True,
)
log.info("Zipapp created with compression")
except TypeError:
# automatically fall back to uncompressed
zipapp.create_archive(
source=tempdir,
target=dest,
interpreter=sys.executable,
)
log.info("Zipapp created without compression")
def _install_deps(tempdir):
"""Install dependencies with pip."""
# TODO we could explicitly pass a python version here
log.info("Installing dependencies")
# apparently pip doesn't have an API, just a cli.
subprocess.check_call(
[
sys.executable,
"-m",
"pip",
"install",
"--requirement",
"requirements.txt",
"--target",
tempdir,
]
)
def generate_version_file(versioning_vars, dest):
log.info("Generating version file")
log.debug("versioning_vars=%r", versioning_vars)
with open(dest, "w") as fh:
print("# GENERATED FILE -- do not edit", file=fh)
for key, value in versioning_vars:
print(f"{key} = {value!r}", file=fh)
def version_kv_pair(value):
if "=" not in value:
raise argparse.ArgumentTypeError(f"not a key=value pair: {value!r}")
key, value = value.split("=", 1)
if key not in _VALID_VERS_VARS:
raise argparse.ArgumentTypeError(f"Unexpected key: {key!r}")
return key, value
def main():
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter("cephadm/build.py: %(message)s"))
log.addHandler(handler)
log.setLevel(logging.INFO)
log.debug("argv: %r", sys.argv)
parser = argparse.ArgumentParser()
parser.add_argument(
"dest", help="Destination path name for new cephadm binary"
)
parser.add_argument(
"--source", help="Directory containing cephadm sources"
)
parser.add_argument(
"--python", help="The path to the desired version of python"
)
parser.add_argument(
"--set-version-var",
"-S",
type=version_kv_pair,
dest="version_vars",
action="append",
help="Set a key=value pair in the generated version info file",
)
args = parser.parse_args()
if not _did_rexec() and args.python:
_reexec(args.python)
log.info(
"Python Version: {v.major}.{v.minor}.{v.micro}".format(
v=sys.version_info
)
)
log.info("Args: %s", vars(args))
if not HAS_ZIPAPP:
# Unconditionally display an error that the version of python
# lacks zipapp (probably too old).
print("error: zipapp module not found", file=sys.stderr)
print(
"(zipapp is available in Python 3.5 or later."
" are you using a new enough version?)",
file=sys.stderr,
)
sys.exit(2)
if args.source:
source = pathlib.Path(args.source).absolute()
else:
source = pathlib.Path(__file__).absolute().parent
dest = pathlib.Path(args.dest).absolute()
log.info("Source Dir: %s", source)
log.info("Destination Path: %s", dest)
_build(dest, source, versioning_vars=args.version_vars)
if __name__ == "__main__":
main()
|
