Move env_setup to pw_env_setup Bug: 67 Change-Id: I285e917968b2e24e0851753ea4802cef1fbe2d67

diff --git a/.gitignore b/.gitignore
index b9b2f70..9c7800a 100644
--- a/.gitignore
+++ b/.gitignore

@@ -42,7 +42,7 @@
 *_REMOTE_*.txt
 
 # Env Setup
-env_setup/.setup.sh
-env_setup/.env_setup.bat
+pw_env_setup/.setup.sh
+pw_env_setup/.env_setup.bat
 .cipd
 .cargo

diff --git a/README.md b/README.md
index cd586ec..a2aa40c 100644
--- a/README.md
+++ b/README.md

@@ -20,14 +20,15 @@
 ```bash
 $ git clone sso://pigweed.googlesource.com/pigweed/pigweed ~/pigweed
 $ cd ~/pigweed
-$ env_setup/cipd/wrapper.py auth-login  # Once per machine.
-$ . env_setup/bootstrap.sh
+# Only need to run auth-login once per machine.
+$ pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py auth-login
+$ . pw_env_setup/bootstrap.sh
 ```
 
-You can use `. env_setup/env_setup.sh` in place of `. env_setup/bootstrap.sh`.
+You can use `. pw_env_setup/setup.sh` in place of `. pw_env_setup/bootstrap.sh`.
 Both should work every time, but `bootstrap.sh` tends to remove and reinstall
-things at the expense of time whereas `env_setup.sh` tends to do basic checks
-to see if time can be saved by skipping expensive operations.
+things at the expense of time whereas `setup.sh` assumes things have
+already been installed and only sets environment variables.
 
 If you're using Homebrew and you get an error saying
 `module 'http.client' has no attribute 'HTTPSConnection'` then your

diff --git a/env_setup/cipd/pigweed.json b/env_setup/cipd/pigweed.json
index 2c6129b..2ce9f0d 120000
--- a/env_setup/cipd/pigweed.json
+++ b/env_setup/cipd/pigweed.json

@@ -1 +1 @@
-../cipd_setup/pigweed.json
\ No newline at end of file
+../../pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json
\ No newline at end of file

diff --git a/env_setup/virtualenv/requirements.txt b/env_setup/virtualenv/requirements.txt
index 3c9f4d0..baf8449 120000
--- a/env_setup/virtualenv/requirements.txt
+++ b/env_setup/virtualenv/requirements.txt

@@ -1 +1 @@
-../virtualenv_setup/requirements.txt
\ No newline at end of file
+../../pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt
\ No newline at end of file

diff --git a/env_setup/bootstrap.sh b/pw_env_setup/bootstrap.sh
similarity index 100%
rename from env_setup/bootstrap.sh
rename to pw_env_setup/bootstrap.sh


diff --git a/env_setup/env_setup.bat b/pw_env_setup/env_setup.bat
similarity index 74%
rename from env_setup/env_setup.bat
rename to pw_env_setup/env_setup.bat
index d53a74b..7152cc1 100644
--- a/env_setup/env_setup.bat
+++ b/pw_env_setup/env_setup.bat

@@ -17,12 +17,12 @@
 
 :: Calls a Powershell script that determines the correct PW_ROOT directory and
 :: exports it as an environment variable.
-for /F "usebackq tokens=1" %%i in (`powershell %%~dp0..\..\env_setup\env_setup.ps1`) do set PW_ROOT=%%i
+for /F "usebackq tokens=1" %%i in (`powershell %%~dp0..\..\pw_env_setup\env_setup.ps1`) do set PW_ROOT=%%i
 
-set shell_file="%PW_ROOT%\env_setup\.env_setup.bat"
+set shell_file="%PW_ROOT%\pw_env_setup\.env_setup.bat"
 
 if not exist %shell_file% (
-  call python %PW_ROOT%\env_setup\env_setup.py --pw-root %PW_ROOT% --shell-file %shell_file%
+  call python %PW_ROOT%\pw_env_setup\py\pw_env_setup\env_setup.py --pw-root %PW_ROOT% --shell-file %shell_file%
 )
 
 call %shell_file%

diff --git a/env_setup/env_setup.ps1 b/pw_env_setup/env_setup.ps1
similarity index 100%
rename from env_setup/env_setup.ps1
rename to pw_env_setup/env_setup.ps1


diff --git a/env_setup/env_setup.sh b/pw_env_setup/env_setup.sh
similarity index 90%
rename from env_setup/env_setup.sh
rename to pw_env_setup/env_setup.sh
index 7e5c16f..a9d18e8 100644
--- a/env_setup/env_setup.sh
+++ b/pw_env_setup/env_setup.sh

@@ -1,4 +1,4 @@
-# Copyright 2019 The Pigweed Authors
+# Copyright 2020 The Pigweed Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
 # use this file except in compliance with the License. You may obtain a copy of
@@ -36,7 +36,7 @@
 PW_ROOT=$(dirname $(dirname $PW_SETUP_SCRIPT_PATH))
 export PW_ROOT
 
-SETUP_SH="$PW_ROOT/env_setup/.setup.sh"
+SETUP_SH="$PW_ROOT/pw_env_setup/.setup.sh"
 
 # Try to use Python 3 if possible by default, before Python 2.
 if which python3 &> /dev/null; then
@@ -50,7 +50,7 @@
   [ $(basename $PW_SETUP_SCRIPT_PATH) = "bootstrap.sh" ] || \
   [ ! -f $SETUP_SH ] || \
   [ ! -s $SETUP_SH ]; then
-  $PYTHON $PW_ROOT/env_setup/env_setup.py --shell-file $SETUP_SH
+  $PYTHON $PW_ROOT/pw_env_setup/py/pw_env_setup/env_setup.py --shell-file $SETUP_SH
 fi
 
 . $SETUP_SH

diff --git a/env_setup/__init__.py b/pw_env_setup/py/pw_env_setup/__init__.py
similarity index 94%
rename from env_setup/__init__.py
rename to pw_env_setup/py/pw_env_setup/__init__.py
index 2c8334f..1b8f5a8 100644
--- a/env_setup/__init__.py
+++ b/pw_env_setup/py/pw_env_setup/__init__.py

@@ -11,3 +11,4 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under
 # the License.
+"""Pigweed environment setup."""

diff --git a/env_setup/cargo_setup/__init__.py b/pw_env_setup/py/pw_env_setup/cargo_setup/__init__.py
similarity index 92%
rename from env_setup/cargo_setup/__init__.py
rename to pw_env_setup/py/pw_env_setup/cargo_setup/__init__.py
index 28a2a34..101d2f3 100644
--- a/env_setup/cargo_setup/__init__.py
+++ b/pw_env_setup/py/pw_env_setup/cargo_setup/__init__.py

@@ -27,8 +27,8 @@
 
     # packages.txt contains packages one per line with two fields: package
     # name and version.
-    package_path = os.path.join(pw_root, 'env_setup', 'cargo_setup',
-                                'packages.txt')
+    package_path = os.path.join(pw_root, 'pw_env_setup', 'py', 'pw_env_setup',
+                                'cargo_setup', 'packages.txt')
     with env(), open(package_path, 'r') as ins:
         for line in ins:
             line = line.strip()

diff --git a/env_setup/cargo_setup/packages.txt b/pw_env_setup/py/pw_env_setup/cargo_setup/packages.txt
similarity index 100%
rename from env_setup/cargo_setup/packages.txt
rename to pw_env_setup/py/pw_env_setup/cargo_setup/packages.txt


diff --git a/env_setup/cipd_setup/.cipd_version b/pw_env_setup/py/pw_env_setup/cipd_setup/.cipd_version
similarity index 100%
rename from env_setup/cipd_setup/.cipd_version
rename to pw_env_setup/py/pw_env_setup/cipd_setup/.cipd_version


diff --git a/env_setup/cipd_setup/.cipd_version.digests b/pw_env_setup/py/pw_env_setup/cipd_setup/.cipd_version.digests
similarity index 100%
rename from env_setup/cipd_setup/.cipd_version.digests
rename to pw_env_setup/py/pw_env_setup/cipd_setup/.cipd_version.digests


diff --git a/env_setup/cipd_setup/__init__.py b/pw_env_setup/py/pw_env_setup/cipd_setup/__init__.py
similarity index 100%
rename from env_setup/cipd_setup/__init__.py
rename to pw_env_setup/py/pw_env_setup/cipd_setup/__init__.py


diff --git a/env_setup/cipd_setup/luci.json b/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json
similarity index 100%
rename from env_setup/cipd_setup/luci.json
rename to pw_env_setup/py/pw_env_setup/cipd_setup/luci.json


diff --git a/env_setup/cipd_setup/pigweed.json b/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json
similarity index 100%
rename from env_setup/cipd_setup/pigweed.json
rename to pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json


diff --git a/env_setup/cipd_setup/update.py b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
similarity index 87%
rename from env_setup/cipd_setup/update.py
rename to pw_env_setup/py/pw_env_setup/cipd_setup/update.py
index eee7d34..5d3c4cf 100755
--- a/env_setup/cipd_setup/update.py
+++ b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py

@@ -29,25 +29,28 @@
 import subprocess
 import sys
 
-SCRIPT_ROOT = os.path.abspath(os.path.dirname(__file__))
-GIT_ROOT = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
-                                   cwd=SCRIPT_ROOT).decode('utf-8').strip()
-
 
 def parse(argv=None):
     """Parse arguments."""
 
+    script_root = os.path.join(os.environ['PW_ROOT'], 'pw_env_setup', 'py',
+                               'pw_env_setup', 'cipd_setup')
+    git_root = subprocess.check_output(
+        ('git', 'rev-parse', '--show-toplevel'),
+        cwd=script_root,
+    ).decode('utf-8').strip()
+
     parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     parser.add_argument(
         '--install-dir',
         dest='root_install_dir',
-        default=os.path.join(GIT_ROOT, '.cipd'),
+        default=os.path.join(git_root, '.cipd'),
     )
     parser.add_argument('--package-file',
                         dest='package_files',
                         action='append')
     parser.add_argument('--cipd',
-                        default=os.path.join(SCRIPT_ROOT, 'wrapper.py'))
+                        default=os.path.join(script_root, 'wrapper.py'))
     parser.add_argument('--cache-dir',
                         default=os.environ.get(
                             'CIPD_CACHE_DIR',
@@ -111,8 +114,15 @@
         env_vars.set('PW_CIPD_INSTALL_DIR', root_install_dir)
         env_vars.set('CIPD_CACHE_DIR', cache_dir)
 
+    pw_root = None
+    if env_vars:
+        pw_root = env_vars.get('PW_ROOT', None)
+    if not pw_root:
+        pw_root = os.environ['PW_ROOT']
+
     # Run cipd for each json file.
-    default_packages = os.path.join(SCRIPT_ROOT, '*.json')
+    default_packages = os.path.join(pw_root, 'pw_env_setup', 'py',
+                                    'pw_env_setup', 'cipd_setup', '*.json')
     for package_file in package_files or glob.glob(default_packages):
         ensure_file = os.path.join(
             root_install_dir,

diff --git a/env_setup/cipd_setup/wrapper.py b/pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py
similarity index 100%
rename from env_setup/cipd_setup/wrapper.py
rename to pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py


diff --git a/env_setup/env_setup.py b/pw_env_setup/py/pw_env_setup/env_setup.py
similarity index 71%
rename from env_setup/env_setup.py
rename to pw_env_setup/py/pw_env_setup/env_setup.py
index f3a3622..755c2f3 100755
--- a/env_setup/env_setup.py
+++ b/pw_env_setup/py/pw_env_setup/env_setup.py

@@ -25,20 +25,52 @@
 from __future__ import print_function
 
 import argparse
+import copy
 import glob
+import inspect
 import os
 import shutil
 import subprocess
 import sys
 
-# TODO(mohrr) remove import-error disabling, not sure why pylint has issues
-# with it.
-import cipd_setup.update  # pylint: disable=import-error
-import cipd_setup.wrapper  # pylint: disable=import-error
-import cargo_setup  # pylint: disable=import-error
-import environment  # pylint: disable=import-error
-import host_build_setup  # pylint: disable=import-error
-import virtualenv_setup  # pylint: disable=import-error
+# TODO(pwbug/67): Remove import hacks once the oxidized prebuilt binaries are
+# proven stable for first-time bootstrapping. For now, continue to support
+# running directly from source without assuming a functioning Python
+# environment when running for the first time.
+
+# If we're running oxidized, filesystem-centric import hacks won't work. In that
+# case, jump straight to the imports and assume oxidation brought in the deps.
+if not getattr(sys, 'oxidized', False):
+    try:
+        # Even if we're running from source, the user may have a functioning
+        # Python environment already set up. Prefer using it over hacks.
+        # pylint: disable=no-name-in-module
+        from pw_env_setup import cargo_setup
+        # pylint: enable=no-name-in-module
+    except ImportError:
+        old_sys_path = copy.deepcopy(sys.path)
+        filename = None
+        if hasattr(sys.modules[__name__], '__file__'):
+            filename = __file__
+        else:
+            # Try introspection in environments where __file__ is not populated.
+            filename = inspect.getfile(inspect.currentframe())
+        # If none of our strategies worked, the imports are going to fail.
+        if filename is None:
+            raise
+        sys.path.append(
+            os.path.abspath(
+                os.path.join(filename, os.path.pardir, os.path.pardir)))
+        import pw_env_setup  # pylint: disable=unused-import
+        sys.path = old_sys_path
+
+# pylint: disable=wrong-import-position
+from pw_env_setup.cipd_setup import update as cipd_update
+from pw_env_setup.cipd_setup import wrapper as cipd_wrapper
+from pw_env_setup import cargo_setup
+from pw_env_setup import environment
+from pw_env_setup import host_build_setup
+from pw_env_setup import virtualenv_setup
 
 
 # TODO(mohrr) remove disable=useless-object-inheritance once in Python 3.
@@ -49,6 +81,8 @@
         super(EnvSetup, self).__init__(*args, **kwargs)
         self._env = environment.Environment()
         self._pw_root = pw_root
+        self._setup_root = os.path.join(pw_root, 'pw_env_setup', 'py',
+                                        'pw_env_setup')
         self._cipd_cache_dir = cipd_cache_dir
         self._shell_file = shell_file
 
@@ -82,11 +116,11 @@
     def cipd(self):
         install_dir = os.path.join(self._pw_root, '.cipd')
 
-        cipd_client = cipd_setup.wrapper.init(install_dir)
+        cipd_client = cipd_wrapper.init(install_dir)
 
         package_files = glob.glob(
-            os.path.join(self._pw_root, 'env_setup', 'cipd_setup', '*.json'))
-        cipd_setup.update.update(
+            os.path.join(self._setup_root, 'cipd_setup', '*.json'))
+        cipd_update.update(
             cipd=cipd_client,
             root_install_dir=install_dir,
             package_files=package_files,
@@ -101,8 +135,8 @@
 
         venv_path = os.path.join(self._pw_root, '.python3-env')
 
-        requirements = os.path.join(self._pw_root, 'env_setup',
-                                    'virtualenv_setup', 'requirements.txt')
+        requirements = os.path.join(self._setup_root, 'virtualenv_setup',
+                                    'requirements.txt')
 
         cipd_bin = os.path.join(
             self._pw_root,

diff --git a/env_setup/environment.py b/pw_env_setup/py/pw_env_setup/environment.py
similarity index 100%
rename from env_setup/environment.py
rename to pw_env_setup/py/pw_env_setup/environment.py


diff --git a/env_setup/environment_test.py b/pw_env_setup/py/pw_env_setup/environment_test.py
similarity index 98%
rename from env_setup/environment_test.py
rename to pw_env_setup/py/pw_env_setup/environment_test.py
index c15e78a..8d083ac 100644
--- a/env_setup/environment_test.py
+++ b/pw_env_setup/py/pw_env_setup/environment_test.py

@@ -29,9 +29,7 @@
 import tempfile
 import unittest
 
-# TODO(mohrr) remove import-error disabling, not sure why pylint has issues
-# with it.
-import environment  # pylint: disable=import-error
+from pw_env_setup import environment
 
 
 class WrittenEnvFailure(Exception):

diff --git a/env_setup/host_build_setup.py b/pw_env_setup/py/pw_env_setup/host_build_setup.py
similarity index 75%
rename from env_setup/host_build_setup.py
rename to pw_env_setup/py/pw_env_setup/host_build_setup.py
index 1bed6b2..b289136 100644
--- a/env_setup/host_build_setup.py
+++ b/pw_env_setup/py/pw_env_setup/host_build_setup.py

@@ -20,7 +20,10 @@
 def install(pw_root, env):
     host_dir = os.path.join(pw_root, 'out', 'host')
     with env():
-        subprocess.check_call(['gn', 'gen', host_dir], cwd=pw_root)
-        subprocess.check_call(['ninja', '-C', host_dir], cwd=pw_root)
+        try:
+            subprocess.check_call(['gn', 'gen', host_dir], cwd=pw_root)
+            subprocess.check_call(['ninja', '-C', host_dir], cwd=pw_root)
+        except subprocess.CalledProcessError:
+            env.echo('warning: host tools failed to build')
 
     env.prepend('PATH', os.path.join(host_dir, 'host_tools'))

diff --git a/env_setup/virtualenv_setup/__init__.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__init__.py
similarity index 100%
rename from env_setup/virtualenv_setup/__init__.py
rename to pw_env_setup/py/pw_env_setup/virtualenv_setup/__init__.py


diff --git a/env_setup/virtualenv_setup/__main__.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
similarity index 84%
rename from env_setup/virtualenv_setup/__main__.py
rename to pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
index e5324c5..385252d 100644
--- a/env_setup/virtualenv_setup/__main__.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py

@@ -17,9 +17,9 @@
 import os
 import sys
 
-# TODO(mohrr) remove import-error disabling, not sure why pylint has issues
-# with it.
-import install  # pylint: disable=import-error
+# TODO(pwbug/67) switch back to 'from pw_env_setup import virtualenv_setup'.
+# from pw_env_setup import virtualenv_setup
+import install as virtualenv_setup  # pylint: disable=import-error
 
 
 def _main():
@@ -42,11 +42,7 @@
                         default=sys.executable,
                         help='Python to use when creating virtualenv.')
 
-    try:
-        install.install(**vars(parser.parse_args()))
-    except install.GitRepoNotFound:
-        print('git repository not found', file=sys.stderr)
-        return -1
+    virtualenv_setup.install(**vars(parser.parse_args()))
 
     return 0
 

diff --git a/env_setup/virtualenv_setup/install.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
similarity index 97%
rename from env_setup/virtualenv_setup/install.py
rename to pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
index 13733b7..c861369 100644
--- a/env_setup/virtualenv_setup/install.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py

@@ -101,9 +101,6 @@
 
     setup_py_files = git_list_files('setup.py', '*/setup.py', cwd=pw_root)
 
-    # Don't install the env_setup "module".
-    setup_py_files = [x for x in setup_py_files if b'env_setup' not in x]
-
     # If not forcing full setup, check if all expected packages are installed,
     # ignoring versions. If they are, skip reinstalling.
     if not full_envsetup:

diff --git a/env_setup/virtualenv_setup/requirements.in b/pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.in
similarity index 100%
rename from env_setup/virtualenv_setup/requirements.in
rename to pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.in


diff --git a/env_setup/virtualenv_setup/requirements.txt b/pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt
similarity index 100%
rename from env_setup/virtualenv_setup/requirements.txt
rename to pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt


diff --git a/env_setup/setup.py b/pw_env_setup/py/setup.py
similarity index 92%
rename from env_setup/setup.py
rename to pw_env_setup/py/setup.py
index c203f5f..9cd86f4 100644
--- a/env_setup/setup.py
+++ b/pw_env_setup/py/setup.py

@@ -30,7 +30,9 @@
     description='Environment setup for Pigweed',
     packages=setuptools.find_packages(),
     test_suite='setup.test_suite',
-    entry_points={'console_scripts': ['_pw_env_setup = env_setup:main']},
+    entry_points={
+        'console_scripts': ['_pw_env_setup = pw_env_setup.env_setup:main'],
+    },
     package_data={
         'pw_env_setup': [
             'cargo_setup/packages.txt',

diff --git a/env_setup/setup.sh b/pw_env_setup/setup.sh
similarity index 100%
rename from env_setup/setup.sh
rename to pw_env_setup/setup.sh


diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
index dc3408a..1377b42 100755
--- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
+++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py

@@ -51,9 +51,11 @@
 #
 def init_cipd(ctx: PresubmitContext):
     # TODO(mohrr) invoke by importing rather than by subprocess.
-    call(sys.executable,
-         ctx.repository_root.joinpath('env_setup/cipd_setup/update.py'),
-         '--install-dir', ctx.output_directory)
+    call(
+        sys.executable,
+        ctx.repository_root.joinpath('pw_env_setup', 'py', 'pw_env_setup',
+                                     'cipd_setup', 'update.py'),
+        '--install-dir', ctx.output_directory)
 
     paths = [ctx.output_directory, ctx.output_directory.joinpath('bin')]
     for base in ctx.output_directory.glob('*'):
@@ -68,7 +70,8 @@
 
 def init_virtualenv(ctx: PresubmitContext):
     """Set up virtualenv, assumes recent Python 3 is already installed."""
-    virtualenv_source = ctx.repository_root.joinpath('env_setup',
+    virtualenv_source = ctx.repository_root.joinpath('pw_env_setup', 'py',
+                                                     'pw_env_setup',
                                                      'virtualenv_setup')
 
     # For speed, don't build the venv if it exists. Use --clean to recreate it.

diff --git a/targets/stm32f429i-disc1/target_docs.rst b/targets/stm32f429i-disc1/target_docs.rst
index 906ce55..462bc87 100644
--- a/targets/stm32f429i-disc1/target_docs.rst
+++ b/targets/stm32f429i-disc1/target_docs.rst

@@ -45,7 +45,7 @@
 .. code:: sh
 
   # Setup pigweed environment.
-  $ . env_setup/setup.sh
+  $ . pw_env_setup/setup.sh
   # Run test.
   $ stm32f429i_disc1_unit_test_runner /path/to/binary
 
@@ -61,7 +61,7 @@
 .. code:: sh
 
   # Setup Pigweed environment.
-  $ . env_setup/setup.sh
+  $ . pw_env_setup/setup.sh
   # Run test.
   $ pw test --root out/disco/ --runner stm32f429i_disc1_unit_test_runner