pull in updated code for faster build

.gitignore

@@ -23,6 +23,7 @@ __pycache__/
 .Python
 venvs/
 ot2monorepoClone/
+ot1temp/
 build/
 develop-eggs/
 dist/

.travis.yml

@@ -9,6 +9,7 @@ install:
   - pip install -e otcustomizers
   - pip install -r protolib/requirements.txt
   - pip install flake8==3.5.0 pytest
+  - make setup
 
 script:
   - python --version
@@ -17,13 +18,14 @@ script:
   # lint all code
   - flake8 protocols/ protolib2/
   - python ./scripts/bad-README-subcategory.py # make sure subcategories don't have 2 spaces
-  # WIP. For now, just make sure this logs without errors in CI.
-  - make install parse-errors parse-ot1 parse-ot2 parse-README
-  - python -m protolib2
+  # combine all parsed files to final zipped JSON in releases/deploy
+  # (NOTE: developers should have run `make all` beforehand and commited the results)
+  - make build
 
 # Deploy the build version in an S3 bucket
 deploy:
   provider: s3
+  region: us-west-2
   access_key_id: $AWS_ACCESS_KEY
   secret_access_key: $AWS_SECRET_KEY
   bucket: protocol-library-builds

CONTRIBUTING.md

@@ -0,0 +1,26 @@
+# Installation / Setup
+
+Run this in your terminal. Pip is required.
+
+WARNING: this will install flake8, pytest, and virtualenv in your current global python environment.
+
+```bash
+pip install -e otcustomizers
+pip install -r protolib/requirements.txt
+pip install flake8==3.5.0 pytest
+make setup
+```
+
+# Generating derived data
+
+To speed up CI, we commit the results of these parsers to git. To run all the parsers in parallel, go to the repository root and from there, do:
+
+`make all -j`
+
+Run this yourself locally before each PR!
+
+Then, **commit** the changed files in `protoBuilds/`.
+
+OT1 and OT2 parsers will **skip** parsing a file when the output JSON file exists and has a more recent timestamp than the source file (eg the output file `protoBuild/exampleProtocol/example.ot2.py.json` corresponds to the source file `protocols/exampleProtocol/example.ot2.py`). If you want to re-parse a file, delete the output file and run the parser again.
+
+If you have problems with the code not matching the website, try doing `make teardown clean setup` to get a fresh setup, then run the parsers again with `make all -j`.

Makefile

@@ -2,56 +2,84 @@ SHELL := /bin/bash
 
 MONOREPO_URI := https://github.com/Opentrons/opentrons.git
 OT1_VERSION := 2.5.2
-OT2_VERSION_TAG := v3.4.0
+OT2_VERSION_TAG := v3.7.0
 OT2_MONOREPO_DIR := ot2monorepoClone
+OT1_TEMP_DIR := ot1temp
+# Parsers output to here
+BUILD_DIR := protoBuilds
 
-OT1_FILE_SUFFIX := *.ot1.py
-OT2_FILE_SUFFIX := *.ot2.py
+# Ignore all protocol dirs that contain a file named '.ignore'
+# on the top protocol folder level
+IGNORED_INPUT_PATHS := $(addsuffix %, $(dir $(wildcard protocols/*/.ignore)))
 
-.PHONY: install
-install:
-	python -m pip install virtualenv
+OT1_INPUT_FILES_UNFILTERED := $(shell find protocols -type f -name '*.ot1.py')
+OT1_INPUT_FILES := $(filter-out $(IGNORED_INPUT_PATHS), $(OT1_INPUT_FILES_UNFILTERED))
+OT1_OUTPUT_FILES := $(patsubst protocols/%.ot1.py, $(BUILD_DIR)/%.ot1.py.json, $(OT1_INPUT_FILES))
+
+OT2_INPUT_FILES_UNFILTERED := $(shell find protocols -type f -name '*.ot2.py')
+OT2_INPUT_FILES := $(filter-out $(IGNORED_INPUT_PATHS), $(OT2_INPUT_FILES_UNFILTERED))
+OT2_OUTPUT_FILES := $(patsubst protocols/%.ot2.py, $(BUILD_DIR)/%.ot2.py.json, $(OT2_INPUT_FILES))
 
-venvs:
-	mkdir venvs/
+.PHONY: all
+all: parse-ot1 parse-ot2 parse-errors parse-README
+	$(MAKE) build
+
+ot2monorepoClone:
+	git clone --depth=1 --branch=$(OT2_VERSION_TAG) $(MONOREPO_URI) $(OT2_MONOREPO_DIR)
+
+.PHONY: setup
+setup:
+	$(MAKE) ot2monorepoClone
+	python -m pip install virtualenv
+	$(MAKE) venvs/ot1 venvs/ot2
 
-venvs/ot1: venvs
-	virtualenv venvs/ot1 && \
+venvs/ot1:
+	mkdir -p venvs
+	virtualenv venvs/ot1
 	source venvs/ot1/bin/activate && \
 	pip install opentrons==$(OT1_VERSION) && \
 	pip install -e otcustomizers && \
 	deactivate
 
+venvs/ot2:
+	mkdir -p venvs
+	virtualenv venvs/ot2
+	source venvs/ot2/bin/activate && \
+	pip install -e otcustomizers && \
+	pip install pipenv && \
+	pushd $(OT2_MONOREPO_DIR)/api/ && \
+	$(MAKE) install && \
+	python setup.py install && \
+	popd && \
+	deactivate
+
 .PHONY: parse-errors
 parse-errors:
 	python protolib2/traverse_errors.py
 
+# TODO: Ian 2019-03-11 maybe ot1 can be parallelized somehow.
+# right now it deletes `configurations.json` and runs into race conditions
+# deleting that file after another process just deleted it
 .PHONY: parse-ot1
-parse-ot1: venvs/ot1
-	source venvs/ot1/bin/activate && \
-	python protolib2/traverse_ot1.py && \
-	deactivate
+parse-ot1: $(OT1_OUTPUT_FILES)
 
-ot2monorepoClone:
-	git clone --depth=1 --branch=$(OT2_VERSION_TAG) $(MONOREPO_URI) $(OT2_MONOREPO_DIR)
-
-venvs/ot2: ot2monorepoClone
-	virtualenv venvs/ot2 && \
-	source venvs/ot2/bin/activate && \
-	pip install -r $(OT2_MONOREPO_DIR)/api/requirements.txt && \
-	pip install -e otcustomizers && \
-	pushd $(OT2_MONOREPO_DIR)/api/ && \
-	python setup.py install && \
-	popd && \
+# Parse all OT1 python files
+$(BUILD_DIR)/%.ot1.py.json: protocols/%.ot1.py
+	mkdir -p $(dir $@)
+	source venvs/ot1/bin/activate && \
+	APP_DATA_DIR=$(OT1_TEMP_DIR)/$@ python protolib2/parse/parseOT1.py $< $@ && \
 	deactivate
 
-# OVERRIDE_SETTINGS_DIR must be set to use opentrons v3
-# (otherwise it will try to access /data dir during 'import opentrons')
 .PHONY: parse-ot2
-parse-ot2: venvs/ot2
+parse-ot2: $(OT2_OUTPUT_FILES)
+
+# Parse all OT2 python files
+# Note: OVERRIDE_SETTINGS_DIR must be set to use opentrons v3
+$(BUILD_DIR)/%.ot2.py.json: protocols/%.ot2.py
+	mkdir -p $(dir $@)
 	source venvs/ot2/bin/activate && \
 	export OVERRIDE_SETTINGS_DIR=$(OT2_MONOREPO_DIR)/api/tests/opentrons/data && \
-  python protolib2/traverse_ot2.py && \
+	python protolib2/parse/parseOT2.py $< $@ && \
 	deactivate
 
 .PHONY: parse-README
@@ -60,4 +88,13 @@ parse-README:
 
 .PHONY: clean
 clean:
-	rm -rf $(OT2_MONOREPO_DIR) venvs
+	rm -rf $(BUILD_DIR)
+
+.PHONY: teardown
+teardown:
+	rm -rf $(OT2_MONOREPO_DIR) venvs $(OT1_TEMP_DIR)
+
+# Take all files in BUILD_DIR and make a single zipped JSON
+.PHONY: build
+build:
+	python -m protolib2

protolib/cli.py

@@ -50,7 +50,7 @@ def prepare_dirs(
 def build_protocol_library():
     parser = configure_parser()
     parsed_input = parser.parse_args()
-    PROTOCOLS_BUILD_DIR = os.path.join(tempfile.gettempdir(), 'proto-builds')
+    PROTOCOLS_BUILD_DIR = os.path.join(tempfile.gettempdir(), 'protoBuilds')
     PROTOCOLS_RELEASE_DIR = parsed_input.library_output_path
 
     print('preparing build & release dirs...')

protolib2/parse/parseOT1.py

@@ -1,5 +1,5 @@
 from inspect import signature, Parameter
-# import json
+import json
 import time
 import shutil
 from opentrons import robot, containers
@@ -8,14 +8,7 @@
 from opentrons.util.environment import settings as opentrons_settings
 
 import sys
-import opentrons
 
-allProtocolFiles = sys.argv[1:]
-
-print('Opentrons verson: ', opentrons.__version__)
-print('Parsing OT1. Files:')
-print(allProtocolFiles)
-print('*-' * 40)
 # HACK to get pipette type
 pipetteType = type(BasePipette(robot, 'a'))
 
@@ -167,3 +160,13 @@ def get_instruments(robot):
          }
         for axis, instr in robot.get_instruments()
     ]
+
+
+if __name__ == '__main__':
+    sourceFilePath = sys.argv[1]
+    destFilePath = sys.argv[2]
+    print('OT1: parsing {} to {}'.format(sourceFilePath, destFilePath))
+
+    result = parse(sourceFilePath)
+    with open(destFilePath, 'w') as f:
+        json.dump(result, f)

protolib2/parse/parseOT2.py

@@ -1,16 +1,9 @@
+import json
 from inspect import signature, Parameter
 import time
 import sys
-import opentrons
 from opentrons import robot, labware, modules
-from opentrons.instruments import Pipette as BasePipette
-
-allProtocolFiles = sys.argv[1:]
-
-print('Opentrons verson: ', opentrons.__version__)
-print('Parsing OT2. Files:')
-print(allProtocolFiles)
-print('*-' * 40)
+from opentrons.legacy_api.instruments import Pipette as BasePipette
 
 all_labware = []
 all_modules = []
@@ -161,3 +154,13 @@ def get_instruments(robot):
          }
         for axis, instr in robot.get_instruments()
     ]
+
+
+if __name__ == '__main__':
+    sourceFilePath = sys.argv[1]
+    destFilePath = sys.argv[2]
+    print('OT2: parsing {} to {}'.format(sourceFilePath, destFilePath))
+
+    result = parse(sourceFilePath)
+    with open(destFilePath, 'w') as f:
+        json.dump(result, f)

protolib2/traversals/__init__.py

@@ -3,7 +3,7 @@
 
 PROTOCOL_DIR = 'protocols'
 RELEASES_DIR = 'releases'
-PROTOCOLS_BUILD_DIR = os.path.join(RELEASES_DIR, 'proto-builds')
+PROTOCOLS_BUILD_DIR = 'protoBuilds'
 ARGS = sys.argv[2::]