Improved Local Installs (#744)

Local installs are now installed to a gitignored folder in the repo's file structure and use the same files as the repository.

This is a minimal flow change with no effect on the Dockerized environment. [skip ci]

.gitignore

@@ -31,4 +31,7 @@ pdks/
 # Misc
 /resolved_version
 
-docker_build/
+/docker_build/
+
+# Local Install
+/install

Makefile

@@ -65,7 +65,16 @@ PRINT_REM_DESIGNS_TIME ?= 0
 SKYWATER_COMMIT ?= $(shell python3 ./dependencies/tool.py sky130 -f commit)
 OPEN_PDKS_COMMIT ?= $(shell python3 ./dependencies/tool.py open_pdks -f commit)
 
-ENV_COMMAND ?= docker run --rm -v $(OPENLANE_DIR):/openlane -v $(PDK_ROOT):$(PDK_ROOT) -e PDK_ROOT=$(PDK_ROOT) $(DOCKER_OPTIONS) $(OPENLANE_IMAGE_NAME)
+# designs is mounted over install so env.tcl is not found inside the Docker
+# container.
+ENV_START = docker run --rm\
+	-v $(OPENLANE_DIR):/openlane\
+	-v $(OPENLANE_DIR)/designs:/openlane/install\
+	-v $(PDK_ROOT):$(PDK_ROOT)\
+	-e PDK_ROOT=$(PDK_ROOT)\
+	$(DOCKER_OPTIONS)
+
+ENV_COMMAND = $(ENV_START) $(OPENLANE_IMAGE_NAME)
 
 ifndef PDK_ROOT
 $(error PDK_ROOT is undefined, please export it before running make)
@@ -184,11 +193,7 @@ pull-openlane:
 .PHONY: mount
 mount:
 	cd $(OPENLANE_DIR) && \
-		docker run -it --rm \
-			-e PDK_ROOT=$(PDK_ROOT) \
-			-v $(OPENLANE_DIR):/openlane \
-			-v $(PDK_ROOT):$(PDK_ROOT) \
-			$(DOCKER_OPTIONS) $(OPENLANE_IMAGE_NAME)
+		$(ENV_START) -ti $(OPENLANE_IMAGE_NAME)
 
 MISC_REGRESSION_ARGS=
 .PHONY: regression regression_test

dependencies/verify_versions.py

@@ -126,11 +126,11 @@ def verify_versions(no_tools: bool = False, report_file=sys.stderr):
     if no_tools:
         return mismatches
 
-    installed_versions_path = join(openlane_dir, "build", "versions")
-    installed = pathlib.Path(installed_versions_path).is_dir()
+    installed = os.getenv("OPENLANE_LOCAL_INSTALL") == "1"
     environment_manifest = None
     if installed:
         # 3a. Compare with installed versions
+        installed_versions_path = join(os.environ["OL_INSTALL_DIR"], "build", "versions")
         environment_manifest = []
         for tool in os.listdir(installed_versions_path):
             protocol, url, commit = open(join(installed_versions_path, tool)).read().split(':')

docs/source/local_installs.md

@@ -1,4 +1,4 @@
-**NOTE: It is far more complex to set up OpenLane without a Docker container. If you are a novice user, it is recommended to use the Docker container.**
+**NOTE: It is more complex to set up OpenLane without a Docker container. If you are a novice user, it is recommended to use the Docker container.**
 
 # Using OpenLane without Docker
 At its core, OpenLane is a set of scripts working with a set of tools. If you'd like to avoid using a Docker container, you can, but you will have to set up all of the tools required by OpenLane on your computer. We do provide a script to assist you with that.
@@ -8,24 +8,17 @@ At its core, OpenLane is a set of scripts working with a set of tools. If you'd
 * pyyaml, venv: `python3 -m pip install pyyaml venv`
 
 # Tool Library
-You can run `python3 ./env.py list-tools` for a list. There are at least a dozen tools to install here. Luckily, you don't have to install them all one-by-one: There is an installation script that installs most of them.
+You can run `python3 ./env.py tool-list` for a list. There are at least a dozen tools to install here. Luckily, you don't have to install them all one-by-one: There is an installation script that installs most of them.
 
-You can invoke `python3 ./env.py local-install`. This tool copies the skeleton and installs all the tools to a directory of your choice, which is by default, `/usr/local/opt/openlane`. Furthermore, if you are on CentOS 7, macOS or Ubuntu 20.04, the installer will offer to install all the required apt, yum or brew packages for you.
+You can invoke `python3 ./env.py local-install`. This tool copies the skeleton and installs all the tools to `$OPENLANE_ROOT_DIR/install`. Furthermore, if you are on CentOS 7, macOS or Ubuntu 20.04, the installer will offer to install all the required apt, yum or brew packages for you.
 
-On macOS, it may be prudent to invoke it as `SKIP_TOOLS=drcu:cugr python3 ./ol_install.py` instead, as these tools are a nightmare to compile on macOS.
+The tools will all be installed with `./install` as a prefix. You'll find all the repos in `./install/build/repos` and a list of versions in `./install/build/versions`.
 
-The tools will all be installed with `/usr/local/opt/openlane` as a prefix. You'll find all the repos in `/usr/local/opt/openlane/build/repos` and a list of versions in `/usr/local/opt/opt/openlane/build/versions`.
+**DO NOTE:** We expect you to bring your own OpenROAD. This installer will make no attempt to install OpenROAD. It's too complex to build in an automated fashion.
 
-**DO NOTE:** We expect you to bring your own OpenROAD. This installer will make no attempt to install OpenROAD.
-
-After the installer is done, you can invoke `sh /usr/local/opt/openlane/openlane <args>` to use OpenLane, where args are the same arguments you'd pass on to `flow.tcl`.
+After the installer is done, you can simply invoke `./flow.tcl` outside of Docker and it should work okay.
 
 # More about how this works
-OpenLane can work as a skeleton with this file structure:
+`flow.tcl` looks for a file called `./install/env.tcl` before it does anything. If it finds it, it sources it. The `./install` directory is aliased in Docker environments, which already have the proper tools installed.
 
-* configuration/
-* scripts/
-* flow.tcl
-* generate_reports.py
-
-You can copy them into any folder, then invoke `tclsh /path/to/flow.tcl` and go to town. The scripts are pretty light on requirements too: You only need Python 3.6+, Perl 5 and Tclsh. Unfortunately, OpenLane cannot accomplish much without its library of open source EDA tooling.
+`./install/env.tcl` contains the necessary environment variables to add the installed tools to PATH and activate the Python virtual environment.

env.py

@@ -25,7 +25,7 @@ import pathlib
 import getpass
 import textwrap
 import subprocess
-from os.path import join, abspath, dirname, exists
+from os.path import join, abspath, dirname, exists, realpath
 from typing import Tuple, Union, List
 
 openlane_dir = dirname(abspath(__file__))
@@ -149,7 +149,7 @@ class Installer(object):
                     {tools["openroad_app"].version_string}
         """))
 
-        install_dir = self.input_default("INSTALL_DIR", "Where do you want to install Openlane?", "/usr/local/opt/openlane")
+        install_dir = realpath("./install")
 
         sh("mkdir", "-p", install_dir, root="retry")
 
@@ -210,7 +210,7 @@ class Installer(object):
 
         install_packages = "no"
         if os_pick != "other":
-            install_packages = self.input_options("INSTALL_PACKAGES", "Do you want to install packages for development?", ["no", "yes"])
+            install_packages = self.input_options("INSTALL_PACKAGES", "Do you want to install dependencies using your package manager?", ["no", "yes"])
         if install_packages != "no":
             def cat_all(dir):
                 result = ""
@@ -241,7 +241,6 @@ class Installer(object):
                     else:
                         arch_packages.append(entry)
 
-
                 sh("pacman", "-S", "--noconfirm", "--needed", *arch_packages, root="retry")
 
                 temp_dir = tempfile.gettempdir()
@@ -252,7 +251,7 @@ class Installer(object):
                         sh("rm", "-rf", "current")
                         sh("git", "clone", package, "current")
                         with chdir("current"):
-                            sh("makepkg", "-si")
+                            sh("makepkg", "-si", "--noconfirm")
             if os_pick == "ubuntu-20.04":
                 raw = cat_all(join(openlane_dir, 'dependencies', 'ubuntu-20.04')).strip().split("\n")
 
@@ -280,7 +279,7 @@ class Installer(object):
         run_env = os.environ.copy()
         run_env["PREFIX"] = install_dir
 
-        path_elements = ["$PATH", "$OL_DIR/bin"]
+        path_elements = ["$OL_INSTALL_DIR/venv/bin", "$OL_INSTALL_DIR/bin"]
 
         if os_pick == "centos-7":
             run_env["CC"] = "/opt/rh/devtoolset-8/root/usr/bin/gcc"
@@ -321,11 +320,7 @@ class Installer(object):
             print("Copying files...")
             for folder in ["bin", "lib", "share", "build", "dependencies"]:
                 sh("mkdir", "-p", folder)
-            copy("configuration")
-            copy("scripts")
-            copy("flow.tcl")
-            copy("dependencies/")
-
+            
             print("Building Python virtual environment...")
             venv_builder = venv.EnvBuilder(clear=True, with_pip=True)
             venv_builder.create("./venv")
@@ -333,9 +328,9 @@ class Installer(object):
             subprocess.run([
                 "bash", "-c", f"""
                     source ./venv/bin/activate
-                    python3 -m pip install --upgrade -r ./dependencies/python/precompile_time.txt
-                    python3 -m pip install --upgrade -r ./dependencies/python/compile_time.txt
-                    python3 -m pip install --upgrade -r ./dependencies/python/run_time.txt
+                    python3 -m pip install --upgrade -r ../dependencies/python/precompile_time.txt
+                    python3 -m pip install --upgrade -r ../dependencies/python/compile_time.txt
+                    python3 -m pip install --upgrade -r ../dependencies/python/run_time.txt
                 """
             ])
 
@@ -382,21 +377,15 @@ class Installer(object):
                         f.write(tool.version_string)
 
             path_elements.reverse()
-            with open("openlane", "w") as f:
+            with open("env.tcl", "w") as f:
                 f.write(textwrap.dedent(f"""\
-                #!/bin/bash
-                OL_DIR="$(dirname "$(test -L "$0" && readlink "$0" || echo "$0")")"
+                set OL_INSTALL_DIR [file dirname [file normalize [info script]]]
 
-                export PATH={":".join(path_elements)}
-
-                FLOW_TCL=${{FLOW_TCL:-$OL_DIR/flow.tcl}}
-                FLOW_TCL=$(realpath $FLOW_TCL)
-
-                source $OL_DIR/venv/bin/activate
-
-                tclsh $FLOW_TCL $@
+                set ::env(OPENLANE_LOCAL_INSTALL) 1
+                set ::env(OL_INSTALL_DIR) "$OL_INSTALL_DIR"
+                set ::env(PATH) "{":".join(path_elements)}:$::env(PATH)"
+                set ::env(VIRTUAL_ENV) "$OL_INSTALL_DIR/venv"
                 """))
-            sh("chmod", "+x", "./openlane")
 
             with open("installed_version", "w") as f:
                 f.write(ol_version)
@@ -404,7 +393,7 @@ class Installer(object):
             install()
 
         print("Done.")
-        print(f"To invoke Openlane from now on, invoke {install_dir}/openlane then pass on the same options you would flow.tcl.")
+        print(f"To invoke Openlane from now on, invoke ./flow.tcl from the OpenLane root without the Makefile.")
 
 # Commands
 def tool_list():

flow.tcl

@@ -15,6 +15,11 @@
 
 
 set ::env(OPENLANE_ROOT) [file dirname [file normalize [info script]]]
+
+if { [file exists $::env(OPENLANE_ROOT)/install/env.tcl ] } {
+	source $::env(OPENLANE_ROOT)/install/env.tcl
+}
+
 if { ! [info exists ::env(OPENROAD_BIN) ] } {
 	set ::env(OPENROAD_BIN) openroad
 }