Create env_setup.py
env_setup.py writes a shell script to be sourced to the provided PATH
that sets up the environment for Pigweed. For now, this new script is
not used, and the previous approach works alongside this new script.
Change-Id: I10bfa91d2482bef72393b70fab4d836f0db8add2
diff --git a/env_setup/cipd/__init__.py b/env_setup/cipd/__init__.py
new file mode 100644
index 0000000..2c8334f
--- /dev/null
+++ b/env_setup/cipd/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
diff --git a/env_setup/cipd/update.py b/env_setup/cipd/update.py
index 1df925e..471a6c0 100755
--- a/env_setup/cipd/update.py
+++ b/env_setup/cipd/update.py
@@ -43,7 +43,8 @@
default=os.path.join(GIT_ROOT, '.cipd'),
)
parser.add_argument('--ensure-file', dest='ensure_files', action='append')
- parser.add_argument('--cipd', default=os.path.join(SCRIPT_ROOT, 'cipd.py'))
+ parser.add_argument('--cipd',
+ default=os.path.join(SCRIPT_ROOT, 'wrapper.py'))
parser.add_argument('--suppress-shell-commands',
action='store_false',
dest='print_shell_commands')
@@ -78,25 +79,42 @@
return False
-def update(cipd, ensure_files, root_install_dir, cache_dir,
- print_shell_commands):
- """Grab the tools listed in ensure_file."""
+def update(
+ cipd,
+ ensure_files,
+ root_install_dir,
+ cache_dir,
+ print_shell_commands,
+ env_vars=None,
+):
+ """Grab the tools listed in ensure_files."""
+ # Set variables used by the wrapper.
+ # TODO(mohrr) remove once transitioned--already configured in new process.
os.environ['CIPD_PY_INSTALL_DIR'] = root_install_dir
os.environ['CIPD_CACHE_DIR'] = cache_dir
if not check_auth(cipd, print_shell_commands):
return
+ # TODO(mohrr) use os.makedirs(..., exist_ok=True).
if not os.path.isdir(root_install_dir):
os.makedirs(root_install_dir)
+ # Save paths for adding to environment (old process).
+ # TODO(mohrr) remove.
paths = [root_install_dir]
env = {
'CIPD_INSTALL_DIR': root_install_dir,
'CIPD_CACHE_DIR': cache_dir,
}
+ if env_vars:
+ env_vars.prepend('PATH', root_install_dir)
+ env_vars.set('CIPD_INSTALL_DIR', root_install_dir)
+ env_vars.set('CIPD_CACHE_DIR', cache_dir)
+
+ # Run cipd for each ensure file.
default_ensures = os.path.join(SCRIPT_ROOT, '*.ensure')
for ensure_file in ensure_files or glob.glob(default_ensures):
install_dir = os.path.join(root_install_dir,
@@ -114,20 +132,35 @@
print(*cmd, file=sys.stderr)
subprocess.check_call(cmd, stdout=sys.stderr)
+ # TODO(mohrr) remove use of paths.
paths.append(install_dir)
paths.append(os.path.join(install_dir, 'bin'))
+ # Set environment variables so tools can later find things under, for
+ # example, 'share'.
name = ensure_file
if os.path.splitext(name)[1] == '.ensure':
name = os.path.splitext(name)[0]
name = os.path.basename(name)
env['{}_CIPD_INSTALL_DIR'.format(name.upper())] = install_dir
+ if env_vars:
+ # Some executables get installed at top-level and some get
+ # installed under 'bin'.
+ env_vars.prepend('PATH', install_dir)
+ env_vars.prepend('PATH', os.path.join(install_dir, 'bin'))
+ env_vars.set('{}_CIPD_INSTALL_DIR'.format(name.upper()),
+ install_dir)
+
+ # TODO(mohrr) remove code from here to end of function.
for path in paths:
print('adding {} to path'.format(path), file=sys.stderr)
paths.append('$PATH')
+ # This block writes environment variables so Pigweed can use tools in
+ # CIPD. It's being replaced with the env_vars object being passed into
+ # this function.
if print_shell_commands:
with tempfile.NamedTemporaryFile(mode='w',
delete=False,
diff --git a/env_setup/cipd/cipd.py b/env_setup/cipd/wrapper.py
similarity index 90%
rename from env_setup/cipd/cipd.py
rename to env_setup/cipd/wrapper.py
index f7f8fe1..13ea378 100755
--- a/env_setup/cipd/cipd.py
+++ b/env_setup/cipd/wrapper.py
@@ -46,9 +46,8 @@
# Get install dir from environment since args cannot be passed through this
# script (args are passed as-is to cipd).
-INSTALL_DIR = os.environ.get('CIPD_PY_INSTALL_DIR',
- os.path.join(SCRIPT_DIR, 'tools'))
-CLIENT = os.path.join(INSTALL_DIR, 'cipd')
+DEFAULT_INSTALL_DIR = os.environ.get('CIPD_PY_INSTALL_DIR',
+ os.path.join(SCRIPT_DIR, 'tools'))
def platform_normalized():
@@ -173,16 +172,16 @@
raise Exception('failed to download client')
-def bootstrap():
+def bootstrap(client):
"""Bootstrap cipd client installation."""
- client_dir = os.path.dirname(CLIENT)
+ client_dir = os.path.dirname(client)
if not os.path.isdir(client_dir):
os.makedirs(client_dir)
print('Bootstrapping cipd client for {}-{}'.format(platform_normalized(),
arch_normalized()))
- tmp_path = os.path.join(INSTALL_DIR, '.cipd.tmp')
+ tmp_path = client + '.tmp'
with open(tmp_path, 'wb') as tmp:
tmp.write(client_bytes())
@@ -194,14 +193,14 @@
'check that digests file is current')
os.chmod(tmp_path, 0o755)
- os.rename(tmp_path, CLIENT)
+ os.rename(tmp_path, client)
-def selfupdate():
+def selfupdate(client):
"""Update cipd client."""
cmd = [
- CLIENT,
+ client,
'selfupdate',
'-version-file', VERSION_FILE,
'-service-url', 'https://{}'.format(CIPD_HOST),
@@ -209,22 +208,24 @@
subprocess.check_call(cmd)
-def init():
+def init(install_dir=DEFAULT_INSTALL_DIR):
"""Install/update cipd client."""
os.environ['CIPD_HTTP_USER_AGENT_PREFIX'] = user_agent()
+ client = os.path.join(install_dir, 'cipd')
+
try:
- if not os.path.isfile(CLIENT):
- bootstrap()
+ if not os.path.isfile(client):
+ bootstrap(client)
try:
- selfupdate()
+ selfupdate(client)
except subprocess.CalledProcessError:
print('CIPD selfupdate failed. Bootstrapping then retrying...',
file=sys.stderr)
- bootstrap()
- selfupdate()
+ bootstrap(client)
+ selfupdate(client)
except Exception:
print('Failed to initialize CIPD. Run '
@@ -232,13 +233,15 @@
"selfupdate -version-file '{version_file}'` "
'to diagnose if this is persistent.'.format(
user_agent=user_agent(),
- client=CLIENT,
+ client=client,
version_file=VERSION_FILE,
),
file=sys.stderr)
raise
+ return client
+
if __name__ == '__main__':
- init()
- subprocess.check_call([CLIENT] + sys.argv[1:])
+ client_exe = init()
+ subprocess.check_call([client_exe] + sys.argv[1:])
diff --git a/env_setup/env_setup.py b/env_setup/env_setup.py
new file mode 100755
index 0000000..5e6dbf8
--- /dev/null
+++ b/env_setup/env_setup.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python
+
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Environment setup script for Pigweed.
+
+This script installs everything and writes out a file for the user's shell
+to source.
+
+For now, this is valid Python 2 and Python 3. Once we switch to running this
+with PyOxidizer it can be upgraded to recent Python 3.
+"""
+
+from __future__ import print_function
+
+import argparse
+import contextlib
+import glob
+import os
+import subprocess
+import sys
+
+import cipd.update
+import cipd.wrapper
+import host_build.init
+import virtualenv.init
+
+
+class UnexpectedAction(ValueError):
+ pass
+
+
+# TODO(mohrr) use attrs.
+class _Action(object): # pylint: disable=useless-object-inheritance
+ # pylint: disable=redefined-builtin,too-few-public-methods
+ def __init__(self, type, name, value, *args, **kwargs):
+ pathsep = kwargs.pop('pathsep', os.pathsep)
+ super(_Action, self).__init__(*args, **kwargs)
+ assert type in ('set', 'prepend', 'append')
+ self.type = type
+ self.name = name
+ self.value = value
+ self.pathsep = pathsep
+
+
+# TODO(mohrr) remove disable=useless-object-inheritance once in Python 3.
+# pylint: disable=useless-object-inheritance
+class Environment(object):
+ """Stores the environment changes necessary for Pigweed.
+
+ These changes can be accessed by writing them to a file for bash-like
+ shells to source or by using this as a context manager.
+ """
+ def __init__(self, *args, **kwargs):
+ pathsep = kwargs.pop('pathsep', os.pathsep)
+ super(Environment, self).__init__(*args, **kwargs)
+ self._actions = []
+ self._pathsep = pathsep
+
+ def set(self, name, value):
+ self._actions.append(_Action('set', name, value))
+
+ def append(self, name, value):
+ self._actions.append(_Action('append', name, value))
+
+ def prepend(self, name, value):
+ self._actions.append(_Action('prepend', name, value))
+
+ def _action_str(self, action):
+ if action.type == 'set':
+ fmt = '{name}="{value}"'
+ elif action.type == 'append':
+ fmt = '{name}="${name}{sep}{value}"'
+ elif action.type == 'prepend':
+ fmt = '{name}="{value}{sep}${name}"'
+ else:
+ raise UnexpectedAction(action.name)
+
+ fmt += '\nexport {name}\n'
+
+ return fmt.format(
+ name=action.name,
+ value=action.value,
+ sep=self._pathsep,
+ )
+
+ def write(self, outs):
+ for action in self._actions:
+ outs.write(self._action_str(action))
+
+ @contextlib.contextmanager
+ def __call__(self):
+ """Set environment as if this was written to a file and sourced."""
+
+ orig_env = os.environ.copy()
+ try:
+ for action in self._actions:
+ if action.type == 'set':
+ os.environ[action.name] = action.value
+ elif action.type == 'append':
+ os.environ[action.name] = self._pathsep.join(
+ os.environ.get(action.name, ''), action.value)
+ elif action.type == 'prepend':
+ os.environ[action.name] = self._pathsep.join(
+ (action.value, os.environ.get(action.name, '')))
+ else:
+ raise UnexpectedAction(action.type)
+ yield self
+
+ finally:
+ for action in self._actions:
+ if action.name in orig_env:
+ os.environ[action.name] = orig_env[action.name]
+ else:
+ os.environ.pop(action.name, None)
+
+
+class EnvSetup(object):
+ """Run environment setup for Pigweed."""
+ def __init__(self, pw_root, cipd_cache_dir, shell_file, *args, **kwargs):
+ super(EnvSetup, self).__init__(*args, **kwargs)
+ self._env = Environment()
+ self._pw_root = pw_root
+ self._cipd_cache_dir = cipd_cache_dir
+ self._shell_file = shell_file
+
+ if isinstance(self._pw_root, bytes):
+ self._pw_root = self._pw_root.decode()
+
+ self._env.set('PW_ROOT', self._pw_root)
+
+ def setup(self):
+ steps = [
+ ('cipd', self.cipd),
+ ('python', self.virtualenv),
+ ('host_tools', self.host_build),
+ ]
+
+ for name, step in steps:
+ print('Setting up {}...\n'.format(name), file=sys.stdout)
+ step()
+ print('\nSetting up {}...done.'.format(name), file=sys.stdout)
+
+ self._env.write(self._shell_file)
+
+ def cipd(self):
+ install_dir = os.path.join(self._pw_root, '.cipd')
+
+ cipd_client = cipd.wrapper.init(install_dir)
+
+ ensure_files = glob.glob(
+ os.path.join(self._pw_root, 'env_setup', 'cipd', '*.ensure'))
+ cipd.update.update(
+ cipd=cipd_client,
+ root_install_dir=install_dir,
+ ensure_files=ensure_files,
+ cache_dir=self._cipd_cache_dir,
+ print_shell_commands=False,
+ env_vars=self._env,
+ )
+
+ def virtualenv(self):
+ venv_path = os.path.join(self._pw_root, '.python3-env')
+
+ requirements = os.path.join(self._pw_root, 'env_setup', 'virtualenv',
+ 'requirements.txt')
+
+ with self._env():
+ # TODO(mohrr) use shutil.which('python3') (Python 3.3+ only).
+ cmd = ['python3', '-c', 'import sys; print(sys.executable)']
+ python = subprocess.check_output(cmd).strip()
+
+ virtualenv.init.init(
+ venv_path=venv_path,
+ requirements=[requirements],
+ python=python,
+ env=self._env,
+ )
+
+ def host_build(self):
+ host_build.init.init(pw_root=self._pw_root, env=self._env)
+
+
+def parse(argv=None):
+ parser = argparse.ArgumentParser()
+
+ try:
+ pw_root = subprocess.check_output(
+ ['git', 'rev-parse', '--show-toplevel']).strip()
+ except subprocess.CalledProcessError:
+ pw_root = None
+ parser.add_argument(
+ '--pw-root',
+ default=pw_root,
+ required=not pw_root,
+ )
+
+ parser.add_argument(
+ '--cipd-cache-dir',
+ default=os.environ.get('CIPD_CACHE_DIR',
+ os.path.expanduser('~/.cipd-cache-dir')),
+ )
+
+ parser.add_argument(
+ '--shell-file',
+ type=argparse.FileType('w'),
+ help='Where to write the file for shells to source.',
+ )
+
+ return parser.parse_args(argv)
+
+
+if __name__ == '__main__':
+ sys.exit(EnvSetup(**vars(parse())).setup())
diff --git a/env_setup/host_build/__init__.py b/env_setup/host_build/__init__.py
new file mode 100644
index 0000000..2c8334f
--- /dev/null
+++ b/env_setup/host_build/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
diff --git a/env_setup/host_build/init.py b/env_setup/host_build/init.py
new file mode 100644
index 0000000..5659a2d
--- /dev/null
+++ b/env_setup/host_build/init.py
@@ -0,0 +1,26 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Builds and sets up environment to use host build."""
+
+import os
+import subprocess
+
+
+def init(pw_root, env):
+ host_dir = os.path.join(pw_root, 'out', 'host')
+ with env():
+ subprocess.check_call(['gn', 'gen', host_dir], cwd=pw_root)
+ subprocess.check_call(['ninja', '-C', host_dir], cwd=pw_root)
+
+ env.prepend('PATH', os.path.join(host_dir, 'host_tools'))
diff --git a/env_setup/virtualenv/__init__.py b/env_setup/virtualenv/__init__.py
new file mode 100644
index 0000000..2c8334f
--- /dev/null
+++ b/env_setup/virtualenv/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
diff --git a/env_setup/virtualenv/init.py b/env_setup/virtualenv/init.py
index 79aa976..2efe17b 100644
--- a/env_setup/virtualenv/init.py
+++ b/env_setup/virtualenv/init.py
@@ -82,6 +82,7 @@
full_envsetup=True,
requirements=(),
python=sys.executable,
+ env=None,
):
"""Creates a venv and installs all packages in this Git repo."""
@@ -140,6 +141,10 @@
pip_install('--log', os.path.join(venv_path, 'pip-packages.log'),
*package_args)
+ if env:
+ env.set('VIRTUAL_ENV', venv_path)
+ env.prepend('PATH', os.path.join(venv_path, 'bin'))
+
def _main():
parser = argparse.ArgumentParser(description=__doc__)
