From 96e54b66bd14c308f528f4380858e23e80e002e5 Mon Sep 17 00:00:00 2001 From: IanLondon Date: Tue, 12 Mar 2019 16:58:38 -0400 Subject: [PATCH] pull in updated code for faster build --- .gitignore | 1 + .travis.yml | 8 ++- CONTRIBUTING.md | 26 +++++++++ Makefile | 97 ++++++++++++++++++++++---------- protolib/cli.py | 2 +- protolib2/parse/parseOT1.py | 19 ++++--- protolib2/parse/parseOT2.py | 21 ++++--- protolib2/traversals/__init__.py | 2 +- 8 files changed, 124 insertions(+), 52 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/.gitignore b/.gitignore index 768fa96394..b04e33b24d 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ __pycache__/ .Python venvs/ ot2monorepoClone/ +ot1temp/ build/ develop-eggs/ dist/ diff --git a/.travis.yml b/.travis.yml index de7da7e7c3..49067abb69 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ install: - pip install -e otcustomizers - pip install -r protolib/requirements.txt - pip install flake8==3.5.0 pytest + - make setup script: - python --version @@ -17,13 +18,14 @@ script: # lint all code - flake8 protocols/ protolib2/ - python ./scripts/bad-README-subcategory.py # make sure subcategories don't have 2 spaces - # WIP. For now, just make sure this logs without errors in CI. - - make install parse-errors parse-ot1 parse-ot2 parse-README - - python -m protolib2 + # combine all parsed files to final zipped JSON in releases/deploy + # (NOTE: developers should have run `make all` beforehand and commited the results) + - make build # Deploy the build version in an S3 bucket deploy: provider: s3 + region: us-west-2 access_key_id: $AWS_ACCESS_KEY secret_access_key: $AWS_SECRET_KEY bucket: protocol-library-builds diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..4c89ae9bba --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,26 @@ +# Installation / Setup + +Run this in your terminal. Pip is required. + +WARNING: this will install flake8, pytest, and virtualenv in your current global python environment. + +```bash +pip install -e otcustomizers +pip install -r protolib/requirements.txt +pip install flake8==3.5.0 pytest +make setup +``` + +# Generating derived data + +To speed up CI, we commit the results of these parsers to git. To run all the parsers in parallel, go to the repository root and from there, do: + +`make all -j` + +Run this yourself locally before each PR! + +Then, **commit** the changed files in `protoBuilds/`. + +OT1 and OT2 parsers will **skip** parsing a file when the output JSON file exists and has a more recent timestamp than the source file (eg the output file `protoBuild/exampleProtocol/example.ot2.py.json` corresponds to the source file `protocols/exampleProtocol/example.ot2.py`). If you want to re-parse a file, delete the output file and run the parser again. + +If you have problems with the code not matching the website, try doing `make teardown clean setup` to get a fresh setup, then run the parsers again with `make all -j`. diff --git a/Makefile b/Makefile index 35af0d7fe9..f9486a2191 100644 --- a/Makefile +++ b/Makefile @@ -2,56 +2,84 @@ SHELL := /bin/bash MONOREPO_URI := https://github.com/Opentrons/opentrons.git OT1_VERSION := 2.5.2 -OT2_VERSION_TAG := v3.4.0 +OT2_VERSION_TAG := v3.7.0 OT2_MONOREPO_DIR := ot2monorepoClone +OT1_TEMP_DIR := ot1temp +# Parsers output to here +BUILD_DIR := protoBuilds -OT1_FILE_SUFFIX := *.ot1.py -OT2_FILE_SUFFIX := *.ot2.py +# Ignore all protocol dirs that contain a file named '.ignore' +# on the top protocol folder level +IGNORED_INPUT_PATHS := $(addsuffix %, $(dir $(wildcard protocols/*/.ignore))) -.PHONY: install -install: - python -m pip install virtualenv +OT1_INPUT_FILES_UNFILTERED := $(shell find protocols -type f -name '*.ot1.py') +OT1_INPUT_FILES := $(filter-out $(IGNORED_INPUT_PATHS), $(OT1_INPUT_FILES_UNFILTERED)) +OT1_OUTPUT_FILES := $(patsubst protocols/%.ot1.py, $(BUILD_DIR)/%.ot1.py.json, $(OT1_INPUT_FILES)) + +OT2_INPUT_FILES_UNFILTERED := $(shell find protocols -type f -name '*.ot2.py') +OT2_INPUT_FILES := $(filter-out $(IGNORED_INPUT_PATHS), $(OT2_INPUT_FILES_UNFILTERED)) +OT2_OUTPUT_FILES := $(patsubst protocols/%.ot2.py, $(BUILD_DIR)/%.ot2.py.json, $(OT2_INPUT_FILES)) -venvs: - mkdir venvs/ +.PHONY: all +all: parse-ot1 parse-ot2 parse-errors parse-README + $(MAKE) build + +ot2monorepoClone: + git clone --depth=1 --branch=$(OT2_VERSION_TAG) $(MONOREPO_URI) $(OT2_MONOREPO_DIR) + +.PHONY: setup +setup: + $(MAKE) ot2monorepoClone + python -m pip install virtualenv + $(MAKE) venvs/ot1 venvs/ot2 -venvs/ot1: venvs - virtualenv venvs/ot1 && \ +venvs/ot1: + mkdir -p venvs + virtualenv venvs/ot1 source venvs/ot1/bin/activate && \ pip install opentrons==$(OT1_VERSION) && \ pip install -e otcustomizers && \ deactivate +venvs/ot2: + mkdir -p venvs + virtualenv venvs/ot2 + source venvs/ot2/bin/activate && \ + pip install -e otcustomizers && \ + pip install pipenv && \ + pushd $(OT2_MONOREPO_DIR)/api/ && \ + $(MAKE) install && \ + python setup.py install && \ + popd && \ + deactivate + .PHONY: parse-errors parse-errors: python protolib2/traverse_errors.py +# TODO: Ian 2019-03-11 maybe ot1 can be parallelized somehow. +# right now it deletes `configurations.json` and runs into race conditions +# deleting that file after another process just deleted it .PHONY: parse-ot1 -parse-ot1: venvs/ot1 - source venvs/ot1/bin/activate && \ - python protolib2/traverse_ot1.py && \ - deactivate +parse-ot1: $(OT1_OUTPUT_FILES) -ot2monorepoClone: - git clone --depth=1 --branch=$(OT2_VERSION_TAG) $(MONOREPO_URI) $(OT2_MONOREPO_DIR) - -venvs/ot2: ot2monorepoClone - virtualenv venvs/ot2 && \ - source venvs/ot2/bin/activate && \ - pip install -r $(OT2_MONOREPO_DIR)/api/requirements.txt && \ - pip install -e otcustomizers && \ - pushd $(OT2_MONOREPO_DIR)/api/ && \ - python setup.py install && \ - popd && \ +# Parse all OT1 python files +$(BUILD_DIR)/%.ot1.py.json: protocols/%.ot1.py + mkdir -p $(dir $@) + source venvs/ot1/bin/activate && \ + APP_DATA_DIR=$(OT1_TEMP_DIR)/$@ python protolib2/parse/parseOT1.py $< $@ && \ deactivate -# OVERRIDE_SETTINGS_DIR must be set to use opentrons v3 -# (otherwise it will try to access /data dir during 'import opentrons') .PHONY: parse-ot2 -parse-ot2: venvs/ot2 +parse-ot2: $(OT2_OUTPUT_FILES) + +# Parse all OT2 python files +# Note: OVERRIDE_SETTINGS_DIR must be set to use opentrons v3 +$(BUILD_DIR)/%.ot2.py.json: protocols/%.ot2.py + mkdir -p $(dir $@) source venvs/ot2/bin/activate && \ export OVERRIDE_SETTINGS_DIR=$(OT2_MONOREPO_DIR)/api/tests/opentrons/data && \ - python protolib2/traverse_ot2.py && \ + python protolib2/parse/parseOT2.py $< $@ && \ deactivate .PHONY: parse-README @@ -60,4 +88,13 @@ parse-README: .PHONY: clean clean: - rm -rf $(OT2_MONOREPO_DIR) venvs + rm -rf $(BUILD_DIR) + +.PHONY: teardown +teardown: + rm -rf $(OT2_MONOREPO_DIR) venvs $(OT1_TEMP_DIR) + +# Take all files in BUILD_DIR and make a single zipped JSON +.PHONY: build +build: + python -m protolib2 diff --git a/protolib/cli.py b/protolib/cli.py index a547a3c1c5..5b6229f2fa 100644 --- a/protolib/cli.py +++ b/protolib/cli.py @@ -50,7 +50,7 @@ def prepare_dirs( def build_protocol_library(): parser = configure_parser() parsed_input = parser.parse_args() - PROTOCOLS_BUILD_DIR = os.path.join(tempfile.gettempdir(), 'proto-builds') + PROTOCOLS_BUILD_DIR = os.path.join(tempfile.gettempdir(), 'protoBuilds') PROTOCOLS_RELEASE_DIR = parsed_input.library_output_path print('preparing build & release dirs...') diff --git a/protolib2/parse/parseOT1.py b/protolib2/parse/parseOT1.py index 37fd7542f3..61858a2294 100644 --- a/protolib2/parse/parseOT1.py +++ b/protolib2/parse/parseOT1.py @@ -1,5 +1,5 @@ from inspect import signature, Parameter -# import json +import json import time import shutil from opentrons import robot, containers @@ -8,14 +8,7 @@ from opentrons.util.environment import settings as opentrons_settings import sys -import opentrons -allProtocolFiles = sys.argv[1:] - -print('Opentrons verson: ', opentrons.__version__) -print('Parsing OT1. Files:') -print(allProtocolFiles) -print('*-' * 40) # HACK to get pipette type pipetteType = type(BasePipette(robot, 'a')) @@ -167,3 +160,13 @@ def get_instruments(robot): } for axis, instr in robot.get_instruments() ] + + +if __name__ == '__main__': + sourceFilePath = sys.argv[1] + destFilePath = sys.argv[2] + print('OT1: parsing {} to {}'.format(sourceFilePath, destFilePath)) + + result = parse(sourceFilePath) + with open(destFilePath, 'w') as f: + json.dump(result, f) diff --git a/protolib2/parse/parseOT2.py b/protolib2/parse/parseOT2.py index 6f882b85b3..c498d07660 100644 --- a/protolib2/parse/parseOT2.py +++ b/protolib2/parse/parseOT2.py @@ -1,16 +1,9 @@ +import json from inspect import signature, Parameter import time import sys -import opentrons from opentrons import robot, labware, modules -from opentrons.instruments import Pipette as BasePipette - -allProtocolFiles = sys.argv[1:] - -print('Opentrons verson: ', opentrons.__version__) -print('Parsing OT2. Files:') -print(allProtocolFiles) -print('*-' * 40) +from opentrons.legacy_api.instruments import Pipette as BasePipette all_labware = [] all_modules = [] @@ -161,3 +154,13 @@ def get_instruments(robot): } for axis, instr in robot.get_instruments() ] + + +if __name__ == '__main__': + sourceFilePath = sys.argv[1] + destFilePath = sys.argv[2] + print('OT2: parsing {} to {}'.format(sourceFilePath, destFilePath)) + + result = parse(sourceFilePath) + with open(destFilePath, 'w') as f: + json.dump(result, f) diff --git a/protolib2/traversals/__init__.py b/protolib2/traversals/__init__.py index e953bfe2fa..f76c5a80ef 100644 --- a/protolib2/traversals/__init__.py +++ b/protolib2/traversals/__init__.py @@ -3,7 +3,7 @@ PROTOCOL_DIR = 'protocols' RELEASES_DIR = 'releases' -PROTOCOLS_BUILD_DIR = os.path.join(RELEASES_DIR, 'proto-builds') +PROTOCOLS_BUILD_DIR = 'protoBuilds' ARGS = sys.argv[2::]