Move Binaries To Docker (#783)

+ Binaries can now either be built or fetched from Docker, not checked into repo history + Tool updater pushes docker images + Added git as a tool (the one in centos repos is very out of date) + Fixed Docker bashrc ~ Fixed cugr, drcu, and vlogtoverilog builds ~ Update CentOS 7 dependencies ~ Folded fetch_submodules into utils.py ~ Assorted Makefile Tweaks
2026-05-29 00:23:55 +08:00 · 2021-12-23 00:35:30 +02:00
parent edcdf49dd4
commit b91f251826
33 changed files with 382 additions and 291 deletions
--- a/.github/scripts/run_tests.py
+++ b/.github/scripts/run_tests.py
@@ -57,6 +57,7 @@ docker_command = [
    shlex.join([
        "python3",
        "run_designs.py",
+        "--tar_list", "all",
        "--disable_timestamp",
        "--tag", test_name,
        "--threads", str(threads_used),
--- a/.github/workflow-documentation.md
+++ b/.github/workflow-documentation.md
@@ -18,10 +18,11 @@ Repository secrets are used to protect certain credentials, but also as reposito

 | Secret      | Description                                                   |
 |---------------|---------------------------------------------------------------|
-| `DOCKER_IMAGE` | The name of the resulting Docker image (minus the tag). |
-| `DOCKERHUB_USER`  | A username for a user that has push access to the organization that owns `DOCKER_IMAGE` on Docker Hub. |
-| `DOCKERHUB_PASSWORD`  | The password/token for the given username that has push access to the organization that owns `DOCKER_IMAGE` on Docker Hub. |
-| `LOG_UPLOAD_INFO` | Information on a cloud platform to upload buckets to, in the format `platform:bucket:encoded_credentials`, where `platform` can be `gcp`/`aws`/etc, `bucket` is the bucket name, and `encoded_credentials` are simply the relevant credentials encoded in base64. It's a bit convoluted, but it makes it so different CI users can switch the platform out by changing a single secret. Currently, only the Google Cloud Platform is supported. Support for other platforms can be added to `upload_log_tarballs.py`. If this secret is not specified, the logs will not be uploaded. |
+| `DOCKER_IMAGE` | The name of the resulting Docker image (minus the tag). In our case, we use `efabless/openlane`. |
+| `TOOL_DOCKER_IMAGE` | The name of the resulting Docker images for tools (minus the tag). In our case, we use `efabless/openlane-tools`. |
+| `DOCKERHUB_USER`  | A username for a user that has push access to the organization that owns `DOCKER_IMAGE` on Docker Hub. In our case, that's an Efabless Employee with push access. |
+| `DOCKERHUB_PASSWORD`  | The password/token for the given username that has push access to the organization that owns `DOCKER_IMAGE` on Docker Hub.|
+| `LOG_UPLOAD_INFO` | **Currently unused**: Information on a cloud platform to upload buckets to, in the format `platform:bucket:encoded_credentials`, where `platform` can be `gcp`/`aws`/etc, `bucket` is the bucket name, and `encoded_credentials` are simply the relevant credentials encoded in base64. It's a bit convoluted, but it makes it so different CI users can switch the platform out by changing a single secret. Currently, only the Google Cloud Platform is supported. Support for other platforms can be added to `upload_log_tarballs.py`. If this secret is not specified, the logs will not be uploaded. |

 ## Tool Updater

--- a/.github/workflows/tool_updater.yml
+++ b/.github/workflows/tool_updater.yml
@@ -30,12 +30,6 @@ jobs:
        with:
          ref: ${{ secrets.MAIN_BRANCH }}

-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
      - name: Export TOOL Name
        run: echo "TOOL=${{ matrix.tools }}" >> $GITHUB_ENV

@@ -44,7 +38,15 @@ jobs:

      - name: Build TOOL
        if: ${{ env.NO_UPDATE != '1' }}
-        run: cd ${GITHUB_WORKSPACE}/docker && make build-${{ env.TOOL }}
+        run: |
+          cd ${GITHUB_WORKSPACE}/docker
+          export TOOL_REPOSITORY=${{ secrets.TOOL_DOCKER_IMAGE }}
+          make build-${{ env.TOOL }}
+      
+      - name: Push Image
+        if: ${{ env.NO_UPDATE != '1' }}
+        run: |
+          docker push ${{ secrets.TOOL_DOCKER_IMAGE }}:$(python3 $(python3 ../dependencies/tool.py --docker-tag-for-os=centos-7)

      - name: Create Pull Request
        if: ${{ env.NO_UPDATE != '1' }}
--- a/45
+++ b/45
@@ -31,6 +31,7 @@ ifeq ($(UNAME_S),Linux)
 DOCKER_OPTIONS += -e DISPLAY=$(DISPLAY) -v /tmp/.X11-unix:/tmp/.X11-unix -v $(HOME)/.Xauthority:/.Xauthority --network host
 endif

+NPROC ?= $(shell getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
 THREADS ?= 1

 ifneq (,$(ROUTING_CORES))
@@ -118,7 +119,7 @@ skywater-library: $(PDK_ROOT)/skywater-pdk
 		git submodule update --init libraries/$(IO_LIBRARY)/latest && \
 		git submodule update --init libraries/$(SPECIAL_VOLTAGE_LIBRARY)/latest && \
 		git submodule update --init libraries/sky130_fd_pr/latest && \
-		$(MAKE) timing
+		$(MAKE) -j$(NPROC) timing

 .PHONY: all-skywater-libraries
 all-skywater-libraries: skywater-pdk
@@ -130,7 +131,7 @@ all-skywater-libraries: skywater-pdk
 		git submodule update --init libraries/sky130_fd_sc_ls/latest && \
 		git submodule update --init libraries/sky130_fd_sc_hvl/latest && \
 		git submodule update --init libraries/sky130_fd_io/latest && \
-		$(MAKE) -j$(THREADS) timing
+		$(MAKE) -j$(NPROC) timing

 ### OPEN_PDKS
 $(PDK_ROOT)/open_pdks:
@@ -139,45 +140,37 @@ $(PDK_ROOT)/open_pdks:
 .PHONY: open_pdks
 open_pdks: $(PDK_ROOT)/ $(PDK_ROOT)/open_pdks
 	cd $(PDK_ROOT)/open_pdks && \
-		git checkout master && git pull && \
+		git checkout master && \
+		git pull && \
 		git checkout -qf $(OPEN_PDKS_COMMIT)

 .PHONY: build-pdk
+native-build-pdk: ENV_COMMAND=env
+native-build-pdk: build-pdk
 build-pdk: $(PDK_ROOT)/open_pdks $(PDK_ROOT)/skywater-pdk
-	[ -d $(PDK_ROOT)/sky130A ] && \
-		(echo "Warning: A sky130A build already exists under $(PDK_ROOT). It will be deleted first!" && \
-		sleep 5 && \
-		rm -rf $(PDK_ROOT)/sky130A) || \
-		true
-	$(ENV_COMMAND) sh -c " cd $(PDK_ROOT)/open_pdks && \
-		./configure --enable-sky130-pdk=$(PDK_ROOT)/skywater-pdk/libraries $(OPEN_PDK_ARGS)"
+	[ -d $(PDK_ROOT)/sky130A ] && rm -rf $(PDK_ROOT)/sky130A || true
+	$(ENV_COMMAND) sh -c "\
+		cd $(PDK_ROOT)/open_pdks && \
+		./configure --enable-sky130-pdk=$(PDK_ROOT)/skywater-pdk/libraries $(OPEN_PDK_ARGS)\
+	"
 	cd $(PDK_ROOT)/open_pdks/sky130 && \
 		$(MAKE) veryclean && \
 		$(MAKE) prerequisites
-	$(ENV_COMMAND) sh -c " cd $(PDK_ROOT)/open_pdks/sky130 && \
+	$(ENV_COMMAND) sh -c "\
+		cd $(PDK_ROOT)/open_pdks/sky130 && \
 		make && \
 		make SHARED_PDKS_PATH=$(PDK_ROOT) install && \
-		make clean"
-
-.PHONY: native-build-pdk
-native-build-pdk: $(PDK_ROOT)/open_pdks $(PDK_ROOT)/skywater-pdk
-	[ -d $(PDK_ROOT)/sky130A ] && \
-		(echo "Warning: A sky130A build already exists under $(PDK_ROOT). It will be deleted first!" && \
-		sleep 5 && \
-		rm -rf $(PDK_ROOT)/sky130A) || \
-		true
-	cd $(PDK_ROOT)/open_pdks && \
-		./configure --enable-sky130-pdk=$(PDK_ROOT)/skywater-pdk/libraries $(OPEN_PDK_ARGS) && \
-		cd sky130 && \
-		$(MAKE) veryclean && \
-		$(MAKE) && \
-		$(MAKE) SHARED_PDKS_PATH=$(PDK_ROOT) install
+		make clean \
+	"

 gen-sources: $(PDK_ROOT)/sky130A
 	touch $(PDK_ROOT)/sky130A/SOURCES
 	OPENLANE_COMMIT=$(git rev-parse HEAD)
 	printf "openlane " > $(PDK_ROOT)/sky130A/SOURCES
 	cd $(OPENLANE_DIR) && git rev-parse HEAD >> $(PDK_ROOT)/sky130A/SOURCES
+	printf "magic " >> $(PDK_ROOT)/sky130A/SOURCES
+	python3 ./dependencies/tool.py -f commit magic >> $(PDK_ROOT)/sky130A/SOURCES
+	printf "\n" >> $(PDK_ROOT)/sky130A/SOURCES
 	printf "skywater-pdk " >> $(PDK_ROOT)/sky130A/SOURCES
 	cd $(PDK_ROOT)/skywater-pdk && git rev-parse HEAD >> $(PDK_ROOT)/sky130A/SOURCES
 	printf "open_pdks " >> $(PDK_ROOT)/sky130A/SOURCES
--- a/dependencies/centos-7/compile_time.txt
+++ b/dependencies/centos-7/compile_time.txt
@@ -43,15 +43,10 @@ ncurses-devel
 ninja-build
 patch
 pcre-devel
-python-devel
-python36u
-python36u-devel
-python36u-libs
-python36u-pip
-python36u-tkinter
+python36-devel
+python36-libs
 qt5-qtbase-devel
 readline-devel
-rh-python35
 strace
 spdlog-devel
 swig3
--- a/dependencies/centos-7/precompile_time.txt
+++ b/dependencies/centos-7/precompile_time.txt
@@ -1,3 +1,2 @@
-https://repo.ius.io/ius-release-el7.rpm
 https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
 centos-release-scl
--- a/dependencies/centos-7/run_time.txt
+++ b/dependencies/centos-7/run_time.txt
@@ -22,9 +22,9 @@ pciutils
 pciutils-libs
 perl
 perl-Switch
-python36u
-python36u-pip
-python36u-tkinter
+python36
+python36-pip
+python36-tkinter
 qt
 qt5-qtbase
 qt5-qtimageformats
--- a/dependencies/tool.py
+++ b/dependencies/tool.py
@@ -59,6 +59,7 @@ def main():
    import argparse
    
    parser = argparse.ArgumentParser(description="Get Tool Info")
+    parser.add_argument("--docker-tag-for-os", default=None)
    parser.add_argument("--docker-args", action="store_true")
    parser.add_argument("--field", "-f")
    parser.add_argument("tool")
@@ -67,7 +68,9 @@ def main():
    
    tool = tools[args.tool]
    
-    if args.docker_args:
+    if args.docker_tag_for_os:
+        print(f"{tool.name}-{tool.commit}-{args.docker_tag_for_os}")
+    elif args.docker_args:
        print(f"--build-arg {tool.name.upper()}_REPO={tool.repo} --build-arg {tool.name.upper()}_COMMIT={tool.commit}", end="")
    elif args.field:
        field = tool.__dict__[args.field]
--- a/dependencies/tool_metadata.yml
+++ b/dependencies/tool_metadata.yml
@@ -58,16 +58,16 @@
    ninja clean
    ninja
    cp padring $PREFIX/bin
- name: vlog2Verilog
+- name: vlogtoverilog
  repo: https://github.com/RTimothyEdwards/qflow
  commit: a550469b63e910ede6e3022e2886bca96462c540
  build: |
-    # Note that vlog2Verilog is part of the qflow repository.
+    # Note that vlogtoverilog is part of the qflow repository.
    ./configure
    cd src
    make clean
-    make -j$(nproc) vlog2Verilog
-    cp vlog2Verilog $PREFIX/bin
+    make -j$(nproc) vlogtoverilog
+    cp vlogtoverilog $PREFIX/bin
 - name: yosys
  repo: https://github.com/YosysHQ/yosys
  commit: d061b0e41a2023b5e72794563b94d6a9b5ab41a1
@@ -79,12 +79,12 @@
 - name: openroad_app
  repo: https://github.com/The-OpenROAD-Project/OpenROAD
  commit: e9d88df58cd6e61dec17d0cd7d355f7d6ec25778
-  build: |
-    mkdir -p ./build
-    cd ./build
-    cmake -DCMAKE_INSTALL_PREFIX=$PREFIX/bin ..
-    make clean
-    make -j$(nproc)
+  build: ''
+  in_install: false
+- name: git
+  repo: https://github.com/git/git
+  commit: e9d7761bb94f20acc98824275e317fa82436c25d
+  build: ''
  in_install: false
 - name: open_pdks
  repo: https://github.com/rtimothyedwards/open_pdks
--- a/dependencies/verify_versions.py
+++ b/dependencies/verify_versions.py
@@ -45,7 +45,8 @@ def verify_versions(no_tools: bool = False, report_file=sys.stderr):

    manifest_names_by_SOURCES_name = {
        "open_pdks": "open_pdks",
-        "skywater": "sky130"
+        "skywater": "sky130",
+        "magic": "magic"
    }
    pdk_manifest_names = set(manifest_names_by_SOURCES_name.values())

@@ -73,6 +74,9 @@ def verify_versions(no_tools: bool = False, report_file=sys.stderr):
            # Format: {tool} {commit}
            
            if sources_str.startswith("-ne"):
+                # Broken file on BSD/macOS where echo -ne is not a thing
+                # Solution: Translate to proper format
+                #
                # Format:
                # -ne {tool}
                # {commit} 
@@ -112,8 +116,8 @@ def verify_versions(no_tools: bool = False, report_file=sys.stderr):
                
                if commit != manifest_commit:
                    mismatches = True
-                    print(f"The version of {manifest_name} installed does not match the one required by the OpenLane flow scripts (installed: {commit}, expected: {manifest_commit})", file=report_file)
-                    print(f"You may want to re-install the PDK by invoking `make pdk`.", file=report_file)
+                    print(f"The version of {manifest_name} used in building PDK does not match the version OpenLane was tested on (installed: {commit}, tested: {manifest_commit})", file=report_file)
+                    print(f"This may introduce some issues. You may want to re-install the PDK by invoking `make pdk`.", file=report_file)

                pdk_manifest_names.add(manifest_name)
        else:
--- a/docker/.gitignore
+++ b/docker/.gitignore
@@ -1,2 +1,4 @@
 build/
-logs/
+logs/
+*/fetch_submodules_from_tarballs.py
+*/utils.py
--- a/docker/Makefile
+++ b/docker/Makefile
@@ -1,19 +1,25 @@
 #DOCKER_BUILD_OPTS ?= --rm --no-cache
-DOCKER_BUILD_OPTS ?= --rm
 DOCKERFILE_PATH ?= ./openlane/Dockerfile
 OPENLANE_IMAGE_NAME ?= efabless/openlane:current
-#ROOT = sudo
+TOOL_REPOSITORY ?= efabless/openlane-tools

-TOOLS = cugr drcu yosys magic openroad_app padring netgen vlogtoverilog cvc
+OS_NAME ?= centos-7
+OS_IMAGE ?= centos:centos7
+
+DOCKER_BUILD_OPTS ?= --rm
+DOCKER_BUILD_INVOCATION ?= docker build # docker buildx build --platform linux/amd64 --load
+BUILD_COMMAND = $(DOCKER_BUILD_INVOCATION) $(DOCKER_BUILD_OPTS)
+
+TOOLS = cugr drcu yosys magic openroad_app padring netgen vlogtoverilog cvc git

 TOOL_BUILD_TARGETS = $(foreach tool,$(TOOLS),build-$(tool))
-TOOL_EXPORT_TARGETS_PHONY = $(foreach tool,$(TOOLS),export-$(tool))
-TOOL_EXPORT_TARGETS_REAL = $(foreach tool,$(TOOLS),tar/$(tool).tar.gz)
+TOOL_EXPORT_TARGETS = $(foreach tool,$(TOOLS),tar/$(tool).tar.gz)
 # ==============================================================================
 # Build Tools
 # ==============================================================================
-build-all: $(TOOL_BUILD_TARGETS) export-all
+all: openlane

+build-all: $(TOOL_BUILD_TARGETS)
 base_image: ./base/Dockerfile
 	cat ../dependencies/centos-7/precompile_time.txt > ./base/yum_precompile_dependencies.txt
 	cat ../dependencies/centos-7/compile_time.txt > ./base/yum_compile_dependencies.txt
@@ -22,27 +28,29 @@ base_image: ./base/Dockerfile
 	cat ../dependencies/python/compile_time.txt > ./base/pip_compile_dependencies.txt
 	cat ../dependencies/python/run_time.txt > ./base/pip_dependencies.txt
 	mkdir -p logs
-	docker build $(DOCKER_BUILD_OPTS) -t openlane-build-base base | tee logs/base.build.txt
+	$(BUILD_COMMAND) -t openlane-build-base --build-arg OS_IMAGE=$(OS_IMAGE) base | tee logs/base.build.txt

 $(TOOL_BUILD_TARGETS): build-% : ./%/Dockerfile base_image
 	mkdir -p logs
-	docker build $(DOCKER_BUILD_OPTS) $(shell python3 ../dependencies/tool.py --docker-args $*) -t $* $* | tee logs/$*.build.txt
-	${MAKE} export-$*
+	cp ./utils.py $*
+	$(BUILD_COMMAND)\
+		$(shell python3 ../dependencies/tool.py --docker-args $*)\
+		-t $(TOOL_REPOSITORY):$(shell python3 ../dependencies/tool.py --docker-tag-for-os=$(OS_NAME) $*)\
+		$* |\
+		tee logs/$*.build.txt

 # ==============================================================================
 # Export Tools
 # ==============================================================================
-.PHONY: $(TOOL_EXPORT_TARGETS)
-$(TOOL_EXPORT_TARGETS_PHONY): export-% :
+$(TOOL_EXPORT_TARGETS): tar/%.tar.gz : FORCE
 	rm -f tar/$*.tar.gz && echo "Deleted existing tar/$*.tar.gz"
 	mkdir -p tar
-	id=$$(${ROOT} docker create $*) ; \
+	python3 ./utils.py pull-if-doesnt-exist $(TOOL_REPOSITORY):$(shell python3 ../dependencies/tool.py --docker-tag-for-os=$(OS_NAME) $*)
+	id=$$(${ROOT} docker create $(TOOL_REPOSITORY):$(shell python3 ../dependencies/tool.py --docker-tag-for-os=$(OS_NAME) $*)) ; \
 	  ${ROOT} docker cp $$id:/build.tar.gz tar/$*.tar.gz ; \
 	  ${ROOT} docker rm -v $$id

-.PHONY: export-all
-export-all: tar/openroad_tools.tar.gz
-tar/openroad_tools.tar.gz: $(TOOL_EXPORT_TARGETS_REAL)
+tar/openroad_tools.tar.gz: $(TOOL_EXPORT_TARGETS)
 	for tarFile in $(foreach tool,$(TOOLS),tar/$(tool).tar.gz); do \
 	  tar -xzf $$tarFile ; \
 	done
@@ -60,10 +68,16 @@ openlane: merge
 merge: tar/openroad_tools.tar.gz tar/openlane.tar.gz ../dependencies/tool_metadata.yml
 	cat ../dependencies/tool_metadata.yml > ./tar/tool_metadata.yml
 	cat ../dependencies/python/run_time.txt > ./tar/pip_dependencies.txt
+	cat ../dependencies/centos-7/precompile_time.txt > ./tar/yum_repos.txt
 	cat ../dependencies/centos-7/run_time.txt > ./tar/yum_dependencies.txt
 	printf "$(shell git rev-parse --short=7 HEAD)" > ./tar/git_version
 	mkdir -p logs/tar
-	docker build --build-arg CACHE_INVALIDATOR=$(shell date +%s) $(DOCKER_BUILD_OPTS) -t $(OPENLANE_IMAGE_NAME) -f $(DOCKERFILE_PATH) ./tar | tee logs/$<.build.txt ; \
+	$(BUILD_COMMAND)\
+		--build-arg CACHE_INVALIDATOR=$(shell date +%s)\
+		--build-arg OS_IMAGE=$(OS_IMAGE)\
+		-t $(OPENLANE_IMAGE_NAME)\
+		-f $(DOCKERFILE_PATH) ./tar\
+		| tee logs/$<.build.txt

 .PHONY: clean_merge
 clean_merge:
--- a/docker/README.md
+++ b/docker/README.md
@@ -15,13 +15,8 @@ openlane
 ```

 ## Building the OpenLane Image
-Tool binaries are currently stored under `tar/`. This was not a good idea, but it's what has happened.
-
-If you want to just build the OpenLane image with the binaries checked into the Repo history, you can invoke:
-
 ```bash
-make merge
-# or alternatively: `make openlane`
+make # or make openlane # or make merge
 ```

 ## Updating a Tool Binary
@@ -34,5 +29,7 @@ make build-<tool_name>
 The following are the available tools:

 ```bash
-cugr drcu yosys magic openroad_app padring netgen vlogtoverilog cvc
-```
+cugr drcu yosys magic openroad_app padring netgen vlogtoverilog cvc git
+```
+
+Be sure to make openlane after.
--- a/docker/base/Dockerfile
+++ b/docker/base/Dockerfile
@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-FROM centos:centos7
+ARG OS_IMAGE="centos:centos7"
+FROM ${OS_IMAGE}

 # Install Yum Dependencies
 COPY ./yum_precompile_dependencies.txt /yum_precompile_dependencies.txt
--- a/docker/cugr/Dockerfile
+++ b/docker/cugr/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2020 Efabless Corporation
+# Copyright 2020-2021 Efabless Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,31 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# syntax = docker/dockerfile:1.0-experimental
-FROM centos:centos7 as build
-
-# # install gcc 8
-RUN yum -y install centos-release-scl && \
-    yum -y install devtoolset-8 devtoolset-8-libatomic-devel
-ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc \
-    CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp \
-    CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ \
-    PATH=/opt/rh/devtoolset-8/root/usr/bin:$PATH \
-    LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:$LD_LIBRARY_PATH
-
-RUN yum install -y git python-devel glibc-static wget
-
-# Installing cmake for build dependency
-RUN wget https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh && \
-    chmod +x cmake-3.14.0-Linux-x86_64.sh  && \
-    ./cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr/local
+FROM openlane-build-base

 # Installing boost for build dependency
-RUN wget https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download && \
-    tar -xf download
-RUN cd boost_1_72_0 && \
+RUN curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download | tar --strip-components=1 -xjC . && \
    ./bootstrap.sh && \
-    ./b2 install --with-system --with-filesystem --with-program_options link=static -j $(nproc)
+    ./b2 install --with-iostreams --with-test --with-serialization --with-system --with-filesystem --with-thread --with-program_options -j $(nproc)

 # needed to get the xxd binaries needed to package the *.dat files
 # into C arrays to be linked directly into the executable
@@ -43,18 +24,16 @@ RUN yum -y install vim-common

 ARG CUGR_REPO
 ARG CUGR_COMMIT
-RUN git clone ${CUGR_REPO} cugr_12012021

-WORKDIR cugr_12012021
+WORKDIR /cugr
+RUN curl -L ${CUGR_REPO}/tarball/${CUGR_COMMIT} | tar -xzC . --strip-components=1

-RUN git checkout ${CUGR_COMMIT}
-
-RUN python scripts/build.py -o release
+RUN python scripts/build.py -o release -t iccad19gr

 RUN mkdir -p /build/bin/ && \
    cp run/iccad19gr /build/bin/cugr

 RUN mkdir -p /build/version
 RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/cugr.version
-RUN git rev-parse HEAD >> /build/version/cugr.version
-RUN tar -czf /build.tar.gz /build
+RUN echo ${CUGR_COMMIT} >> /build/version/cugr.version
+RUN tar -czf /build.tar.gz /build
--- a/docker/drcu/Dockerfile
+++ b/docker/drcu/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2020 Efabless Corporation
+# Copyright 2020-2021 Efabless Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,46 +11,29 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# syntax = docker/dockerfile:1.0-experimental
-FROM centos:centos7 as build
-
-# # install gcc 8
-RUN yum -y install centos-release-scl && \
-    yum -y install devtoolset-8 devtoolset-8-libatomic-devel
-ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc \
-    CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp \
-    CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ \
-    PATH=/opt/rh/devtoolset-8/root/usr/bin:$PATH \
-    LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:$LD_LIBRARY_PATH
-
-RUN yum install -y git python-devel glibc-static wget bzip2
-
-# Installing cmake for build dependency
-RUN wget https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh && \
-    chmod +x cmake-3.14.0-Linux-x86_64.sh  && \
-    ./cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr/local
+FROM openlane-build-base

 # Installing boost for build dependency
-RUN wget https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download && \
-    tar -xf download
-RUN cd boost_1_72_0 && \
+RUN curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download | tar --strip-components=1 -xjC . && \
    ./bootstrap.sh && \
-    ./b2 install --with-system --with-filesystem --with-program_options link=static -j $(nproc)
+    ./b2 install --with-iostreams --with-test --with-serialization --with-system --with-filesystem --with-thread --with-program_options link=static -j $(nproc)
+
+# needed to get the xxd binaries needed to package the *.dat files
+# into C arrays to be linked directly into the executable
+RUN yum -y install vim-common

 ARG DRCU_REPO
 ARG DRCU_COMMIT
-RUN git clone ${DRCU_REPO} drcu_18012021

-WORKDIR drcu_18012021
+WORKDIR /drcu
+RUN curl -L ${DRCU_REPO}/tarball/${DRCU_COMMIT} | tar -xzC . --strip-components=1

-RUN git checkout ${DRCU_COMMIT}
-
-RUN python scripts/build.py -o release
+RUN python scripts/build.py -o release -t ispd19dr

 RUN mkdir -p /build/bin/ && \
    cp run/ispd19dr /build/bin/drcu

 RUN mkdir -p /build/version
 RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/drcu.version
-RUN git rev-parse HEAD >> /build/version/drcu.version
+RUN echo ${DRCU_COMMIT} >> /build/version/drcu.version
 RUN tar -czf /build.tar.gz /build
--- a/docker/git/Dockerfile
+++ b/docker/git/Dockerfile
@@ -0,0 +1,30 @@
+# Copyright 2020-2021 Efabless Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM openlane-build-base
+
+ARG GIT_REPO
+ARG GIT_COMMIT
+
+WORKDIR /git
+RUN curl -L ${GIT_REPO}/tarball/${GIT_COMMIT} |\
+	tar -xzC . --strip-components=1 &&\
+	make configure &&\
+	./configure --prefix=/build &&\
+    make -j$(nproc)&&\
+	make install
+
+RUN mkdir -p /build/version/
+RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/git.version
+RUN echo ${GIT_COMMIT} >> /build/version/git.version
+RUN tar -czf /build.tar.gz /build
--- a/docker/openlane/Dockerfile
+++ b/docker/openlane/Dockerfile
@@ -12,21 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-FROM centos:centos7
+ARG OS_IMAGE="centos:centos7"
+FROM ${OS_IMAGE}

 # Install Yum Dependencies
 COPY ./yum_dependencies.txt /yum_dependencies.txt
-RUN yum install -y https://repo.ius.io/ius-release-el$(rpm -E '%{rhel}').rpm && \
-	yum install -y $(cat /yum_dependencies.txt) && \
+COPY ./yum_repos.txt /yum_repos.txt
+RUN yum install --setopt=skip_missing_names_on_install=False -y $(cat /yum_repos.txt) && \
+	yum install --setopt=skip_missing_names_on_install=False -y $(cat /yum_dependencies.txt) && \
 	yum clean all && \
 	rm -rf /var/cache/yum

 RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 60

-## Update and Configure Git
-RUN yum install -y https://packages.endpoint.com/rhel/7/os/x86_64/endpoint-repo-1.7-1.x86_64.rpm && \
-	yum install -y git
-
+## Set Git
 RUN git config --global user.name "OpenLane User" 
 RUN git config --global user.email "openlane.user@localhost"

--- a/docker/openroad_app/Dockerfile
+++ b/docker/openroad_app/Dockerfile
@@ -46,10 +46,14 @@ RUN git clone -b v1.8.1 https://github.com/gabime/spdlog \
 ARG OPENROAD_APP_REPO
 ARG OPENROAD_APP_COMMIT

+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV LC_CTYPE en_US.UTF-8
+
 WORKDIR /openroad
 RUN curl -L ${OPENROAD_APP_REPO}/tarball/${OPENROAD_APP_COMMIT} | tar -xzC . --strip-components=1
-COPY ./fetch_submodules_from_tarballs.py .
-RUN python3 ./fetch_submodules_from_tarballs.py ${OPENROAD_APP_REPO} ${OPENROAD_APP_COMMIT}
+COPY ./utils.py .
+RUN python3 ./utils.py fetch-submodules-from-tarballs ${OPENROAD_APP_REPO} ${OPENROAD_APP_COMMIT}

 # Build OpenROAD
 RUN sed -i "s/GITDIR-NOTFOUND/${OPENROAD_APP_COMMIT}/" cmake/GetGitRevisionDescription.cmake
--- a/docker/openroad_app/fetch_submodules_from_tarballs.py
+++ b/docker/openroad_app/fetch_submodules_from_tarballs.py
@@ -1,131 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2020-2021 Efabless Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# gitmodules = open(".gitmodules").read()
-import re
-import os
-import sys
-import json
-import shutil
-import pathlib
-import tempfile
-import subprocess
-import urllib.parse
-
-"""
-Must be run from inside an extracted repository tarball.
-
-Given the repository's URL and commit, which are available, a table of the
-git submodules with their repositories, commits and paths is constructed and
-then promptly downloaded and extracted using only the GitHub APIs (and curl),
-no git involved.
-
-This makes things much faster than having to clone an repo's entire history then
-its submodule's entire history.
-"""
-
-args = sys.argv[1:]
-
-if len(args) != 2:
-    print(f"Usage: {__file__} <repository> <commit>", file=sys.stderr)
-    exit(os.EX_USAGE)
-
-repository, commit = args
-
-repository_path_info: urllib.parse.SplitResult = urllib.parse.urlsplit(repository)
-
-# 1. Get Commits Of Submodules
-api_result = None
-
-try:
-    api_result = subprocess.check_output([
-        "curl",
-        "--fail",
-        "-s",
-        "-L",
-        "-H", "Accept: application/vnd.github.v3+json",
-        f"https://api.github.com/repos{repository_path_info.path}/git/trees/{commit}?recursive=True"
-    ])
-except Exception as e:
-    print(e, file=sys.stderr)
-    sys.exit(os.EX_DATAERR)
-
-api_result_parsed = json.loads(api_result)
-api_result_tree = api_result_parsed["tree"]
-submodules = [element for element in api_result_tree if element['type'] == 'commit']
-shas_by_path = { submodule['path']: submodule['sha'] for submodule in submodules }
-
-# 2. Get Submodule Manifest
-api_result = None
-
-try:
-    api_result = subprocess.check_output([
-        "curl",
-        "--fail",
-        "-s",
-        "-L",
-        f"https://raw.githubusercontent.com/{repository_path_info.path}/{commit}/.gitmodules"
-    ])
-except Exception as e:
-    print(e, file=sys.stderr)
-    sys.exit(os.EX_DATAERR)
-
-gitmodules = api_result.decode("utf8")
-
-section_line_rx = re.compile(r"\[\s*submodule\s+\"([\w\-\/]+)\"\]")
-key_value_line_rx = re.compile(r"(\w+)\s*=\s*(.+)")
-
-submodules_by_name = {}
-current = {} # First one is discarded
-for line in gitmodules.split("\n"):
-    section_match = section_line_rx.search(line)
-    if section_match is not None:
-        name = section_match[1]
-        submodules_by_name[name] = {}
-        current = submodules_by_name[name]
-    
-    kvl_match = key_value_line_rx.search(line)
-    if kvl_match is not None:
-        key, value = kvl_match[1], kvl_match[2]
-        current[key] = value
-
-for name, submodule in submodules_by_name.items():
-    submodule["commit"] = shas_by_path.get(submodule["path"])
-    if submodule["url"].endswith(".git"):
-        submodule["url"] = submodule["url"][:-4]
-
-# 3. Extract Submodules
-temp_dir = tempfile.gettempdir()
-for (name, values) in submodules_by_name.items():
-    name_fs = re.sub(r"\/", "_", name)
-    tarball = os.path.join(temp_dir, f"{name_fs}.tar.gz")
-    path = values["path"]
-    url = values["url"]
-    commit = values["commit"]
-
-    url = os.path.join(url, "tarball", commit)
-
-    print(f"Downloading {url} to {path}...", file=sys.stderr)
-    subprocess.check_call([
-        "curl", "-sL", "-o", tarball, url        
-    ])
-
-    shutil.rmtree(path, ignore_errors=True)
-    
-    pathlib.Path(path).mkdir(parents=True, exist_ok=True)
-    
-    subprocess.check_call([
-        "tar", "-xzf", tarball, "--strip-components=1", "-C", path
-    ])
-
--- a/docker/sky130A/Dockerfile
+++ b/docker/sky130A/Dockerfile
@@ -0,0 +1,70 @@
+# Copyright 2020-2021 Efabless Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# WIP, probably will never make it to the image because it's taking forever to build
+
+FROM openlane-build-base AS build
+
+ARG SKY130_REPO
+ARG SKY130_COMMIT
+
+ENV PDK_ROOT /build/pdk
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV LC_CTYPE en_US.UTF-8
+
+WORKDIR ${PDK_ROOT}
+RUN git clone ${SKY130_REPO}    
+WORKDIR ${PDK_ROOT}/skywater-pdk
+RUN git checkout main &&\
+    git submodule init &&\
+    git pull --no-recurse-submodules &&\
+	git checkout -qf ${SKY130_COMMIT}
+RUN git submodule update --init libraries/sky130_fd_sc_hd/latest &&\
+    git submodule update --init libraries/sky130_fd_sc_hs/latest &&\
+    git submodule update --init libraries/sky130_fd_sc_hdll/latest &&\
+    git submodule update --init libraries/sky130_fd_sc_ms/latest &&\
+    git submodule update --init libraries/sky130_fd_sc_ls/latest &&\
+    git submodule update --init libraries/sky130_fd_sc_hvl/latest &&\
+    git submodule update --init libraries/sky130_fd_io/latest
+RUN make -j$(nproc) timing > /build/pdk_timing.log 2>&1
+
+ARG MAGIC_REPO
+ARG MAGIC_COMMIT
+
+WORKDIR /magic
+RUN curl -L ${MAGIC_REPO}/tarball/${MAGIC_COMMIT} | tar -xzC . --strip-components=1 && \
+    ./configure --prefix=/usr && \
+    make -j$(nproc) && \
+    make install
+
+ARG OPEN_PDKS_REPO
+ARG OPEN_PDKS_COMMIT
+
+WORKDIR ${PDK_ROOT}
+RUN git clone ${OPEN_PDKS_REPO}
+WORKDIR ${PDK_ROOT}/open_pdks
+RUN git checkout master &&\
+	git checkout -qf ${OPEN_PDKS_COMMIT}
+RUN ./configure --enable-sky130-pdk=${PDK_ROOT}/skywater-pdk/libraries --enable-sram-sky130
+WORKDIR ${PDK_ROOT}/open_pdks/sky130
+RUN make alpha-repo xschem-repo sram-repo > /build/pdk_prereq.log 2>&1 
+RUN make > ./build/pdk.log 2>&1
+RUN make SHARED_PDKS_PATH=${PDK_ROOT} install
+
+
+
+# ---
+FROM openlane-build-base 
+
--- a/docker/tar/.gitignore
+++ b/docker/tar/.gitignore
@@ -1,7 +1,4 @@
-openroad_build.tar.gz
-openroad_tools.tar.gz
-openlane.tar.gz
-openLANE_flow.tar.gz
+/*.tar.gz
 /*.yml
 /*.txt
 /git_version
--- a/docker/tar/cugr.tar.gz
+++ b/docker/tar/cugr.tar.gz
--- a/docker/tar/cvc.tar.gz
+++ b/docker/tar/cvc.tar.gz
--- a/docker/tar/drcu.tar.gz
+++ b/docker/tar/drcu.tar.gz
--- a/docker/tar/magic.tar.gz
+++ b/docker/tar/magic.tar.gz
--- a/docker/tar/netgen.tar.gz
+++ b/docker/tar/netgen.tar.gz
--- a/docker/tar/openroad_app.tar.gz
+++ b/docker/tar/openroad_app.tar.gz
--- a/docker/tar/padring.tar.gz
+++ b/docker/tar/padring.tar.gz
--- a/docker/tar/route.tar.gz
+++ b/docker/tar/route.tar.gz
--- a/docker/tar/vlogtoverilog.tar.gz
+++ b/docker/tar/vlogtoverilog.tar.gz
--- a/docker/tar/yosys.tar.gz
+++ b/docker/tar/yosys.tar.gz
--- a/docker/utils.py
+++ b/docker/utils.py
@@ -0,0 +1,148 @@
+import re
+import os
+import sys
+import json
+import click
+import shutil
+import pathlib
+import tempfile
+import subprocess
+import urllib.parse
+
+@click.group()
+def cli():
+    pass
+
+@click.command("pull-if-doesnt-exist")
+@click.argument("image")
+def pull_if_doesnt_exist(image):
+    images = subprocess.check_output([
+        "docker",
+        "images",
+        image
+    ]).decode("utf8").rstrip().split("\n")[1:]
+    if len(images) < 1:
+        print(f"{image} not found, pulling...")
+        subprocess.check_call([
+            "docker",
+            "pull",
+            image
+        ])
+cli.add_command(pull_if_doesnt_exist)
+
+@click.command("fetch-submodules-from-tarballs")
+@click.option("--filter", default=".", help="regular expression to match submodule paths")
+@click.argument("repository")
+@click.argument("commit")
+def fetch_submodules_from_tarballs(filter, repository, commit):
+    """
+    Must be run from inside an extracted repository tarball.
+
+    Given the repository's URL and commit, which are available, a table of the
+    git submodules with their repositories, commits and paths is constructed and
+    then promptly downloaded and extracted using only the GitHub APIs (and curl),
+    no git involved.
+
+    This makes things much faster than having to clone an repo's entire history then
+    its submodule's entire history.
+    """
+
+    repository_path_info: urllib.parse.SplitResult = urllib.parse.urlsplit(repository)
+
+    # 1. Get Commits Of Submodules
+    api_result = None
+
+    try:
+        api_result = subprocess.check_output([
+            "curl",
+            "--fail",
+            "-s",
+            "-L",
+            "-H", "Accept: application/vnd.github.v3+json",
+            f"https://api.github.com/repos{repository_path_info.path}/git/trees/{commit}?recursive=True"
+        ])
+    except Exception as e:
+        print(e, file=sys.stderr)
+        sys.exit(os.EX_DATAERR)
+
+    api_result_parsed = json.loads(api_result)
+    api_result_tree = api_result_parsed["tree"]
+    submodules = [element for element in api_result_tree if element['type'] == 'commit']
+    shas_by_path = { submodule['path']: submodule['sha'] for submodule in submodules }
+
+    # 2. Get Submodule Manifest
+    api_result = None
+
+    try:
+        api_result = subprocess.check_output([
+            "curl",
+            "--fail",
+            "-s",
+            "-L",
+            f"https://raw.githubusercontent.com/{repository_path_info.path}/{commit}/.gitmodules"
+        ])
+    except Exception as e:
+        print(e, file=sys.stderr)
+        sys.exit(os.EX_DATAERR)
+
+    gitmodules = api_result.decode("utf8")
+
+    section_line_rx = re.compile(r"\[\s*submodule\s+\"([\w\-\.\/]+)\"\]")
+    key_value_line_rx = re.compile(r"(\w+)\s*=\s*(.+)")
+
+    submodules_by_name = {}
+    current = {} # First one is discarded
+    for line in gitmodules.split("\n"):
+        section_match = section_line_rx.search(line)
+        if section_match is not None:
+            name = section_match[1]
+            submodules_by_name[name] = {}
+            current = submodules_by_name[name]
+        
+        kvl_match = key_value_line_rx.search(line)
+        if kvl_match is not None:
+            key, value = kvl_match[1], kvl_match[2]
+            current[key] = value
+
+    for name, submodule in submodules_by_name.items():
+        submodule["commit"] = shas_by_path.get(submodule["path"])
+        if submodule["url"].endswith(".git"):
+            submodule["url"] = submodule["url"][:-4]
+
+    # 3. Extract Submodules
+    temp_dir = tempfile.gettempdir()
+    filter_rx = re.compile(filter, flags=re.I)
+    for (name, values) in submodules_by_name.items():
+        path = values["path"]
+
+        if filter_rx.match(path) is None:
+            print(f"Skipping {path}...", flush=True)
+            continue
+        else:
+            print(f"Expanding {path}...", flush=True)
+
+        name_fs = re.sub(r"\/", "_", name)
+        tarball = os.path.join(temp_dir, f"{name_fs}.tar.gz")
+
+        url = values["url"]
+        commit = values["commit"]
+
+        url = os.path.join(url, "tarball", commit)
+
+        print(f"Downloading {url} to {path}...", file=sys.stderr)
+        subprocess.check_call([
+            "curl", "-sL", "-o", tarball, url        
+        ])
+
+        shutil.rmtree(path, ignore_errors=True)
+        
+        pathlib.Path(path).mkdir(parents=True, exist_ok=True)
+        
+        subprocess.check_call([
+            "tar", "-xzf", tarball, "--strip-components=1", "-C", path
+        ])
+cli.add_command(fetch_submodules_from_tarballs)
+
+
+if __name__ == '__main__':
+    cli()