Skip to content

Commit

Permalink
[CoreEngine] add the option to disable ota when launching a job.
Browse files Browse the repository at this point in the history
  • Loading branch information
@fedml-alex
fedml-alex committed Nov 3, 2023
1 parent cdd73b5 commit e29ac57
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 70 deletions.
137 changes: 69 additions & 68 deletions devops/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,24 +227,24 @@ pipeline {

sh './devops/scripts/build-fedml-docker.sh'

sh 'docker build --network=host -f ./devops/dockerfile/device-image/Dockerfile-Base -t public.ecr.aws/x6k8q1x9/fedml-device-image:base .'

sh 'docker build --network=host -f ./devops/dockerfile/device-image/Dockerfile-Release -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_DEVICE_IMAGE:$VERSION_PRODUCTION .'

sh 'docker build --network=host -f ./devops/dockerfile/server-agent/Dockerfile-Release -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_SERVER_AGENT:$VERSION_PRODUCTION .'

sh 'docker build --network=host -f ./devops/dockerfile/client-agent/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_CLIENT_AGENT:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'

sh 'docker build --network=host -f ./devops/dockerfile/edge-server/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_EDGE_SERVER:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'

sh 'docker build --network=host -f ./devops/dockerfile/docs-server/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_DOCS_SERVER:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'

sh 'docker build --network=host -f ./devops/dockerfile/edge-client-server/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_EDGE_CLIENT_SERVER:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'

sh 'docker build --network=host -f ./devops/dockerfile/model-inference-backend/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_BACKEND:latest .'
sh 'docker build --network=host -f ./devops/dockerfile/model-inference-ingress/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_INGRESS:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'
sh 'docker build --network=host -f ./devops/dockerfile/model-premise-master/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_MASTER:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'
sh 'docker build --network=host -f ./devops/dockerfile/model-premise-slave/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_SLAVE:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'
// sh 'docker build --network=host -f ./devops/dockerfile/device-image/Dockerfile-Base -t public.ecr.aws/x6k8q1x9/fedml-device-image:base .'
//
// sh 'docker build --network=host -f ./devops/dockerfile/device-image/Dockerfile-Release -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_DEVICE_IMAGE:$VERSION_PRODUCTION .'
//
// sh 'docker build --network=host -f ./devops/dockerfile/server-agent/Dockerfile-Release -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_SERVER_AGENT:$VERSION_PRODUCTION .'
//
// sh 'docker build --network=host -f ./devops/dockerfile/client-agent/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_CLIENT_AGENT:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'
//
// sh 'docker build --network=host -f ./devops/dockerfile/edge-server/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_EDGE_SERVER:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'
//
// sh 'docker build --network=host -f ./devops/dockerfile/docs-server/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_DOCS_SERVER:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'
//
// sh 'docker build --network=host -f ./devops/dockerfile/edge-client-server/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_EDGE_CLIENT_SERVER:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'
//
// sh 'docker build --network=host -f ./devops/dockerfile/model-inference-backend/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_BACKEND:latest .'
// sh 'docker build --network=host -f ./devops/dockerfile/model-inference-ingress/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_INGRESS:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'
// sh 'docker build --network=host -f ./devops/dockerfile/model-premise-master/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_MASTER:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'
// sh 'docker build --network=host -f ./devops/dockerfile/model-premise-slave/Dockerfile -t $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_SLAVE:$VERSION_PRODUCTION --build-arg VERSION=$VERSION_PRODUCTION .'
}
}
}
Expand Down Expand Up @@ -363,46 +363,46 @@ pipeline {
sh './devops/scripts/push-fedml-docker.sh'
}

withCredentials([usernamePassword(passwordVariable : 'AWS_IAM_ACCESS_ID' ,usernameVariable : 'AWS_ECR_USERNAME' ,credentialsId : "$AWS_ECR_CREDENTIAL_ID_PRODUCTION" ,)]) {
withCredentials([usernamePassword(passwordVariable : 'AWS_IAM_ACCESS_KEY' ,usernameVariable : 'AWS_REGION' ,credentialsId : "$AWS_IAM_CREDENTIAL_ID_PRODUCTION" ,)]) {
sh 'curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"'
sh 'unzip -o awscliv2.zip'
sh './aws/install'
sh 'chmod a+x ./devops/scripts/set-aws-credentials.sh;./devops/scripts/set-aws-credentials.sh $AWS_IAM_ACCESS_ID $AWS_IAM_ACCESS_KEY $AWS_REGION'
sh 'docker login --username $AWS_ECR_USERNAME --password $(/usr/local/bin/aws ecr-public get-login-password --region $AWS_REGION) $REGISTRY_PRODUCTION'
}
}

sh 'docker push public.ecr.aws/x6k8q1x9/fedml-device-image:base'
sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_DEVICE_IMAGE:$VERSION_PRODUCTION'
sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_SERVER_AGENT:$VERSION_PRODUCTION'
sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_CLIENT_AGENT:$VERSION_PRODUCTION'
sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_EDGE_SERVER:$VERSION_PRODUCTION'
sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_DOCS_SERVER:$VERSION_PRODUCTION'

sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_BACKEND:latest'
sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_INGRESS:$VERSION_PRODUCTION'
sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_MASTER:$VERSION_PRODUCTION'
sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_SLAVE:$VERSION_PRODUCTION'

withCredentials([usernamePassword(passwordVariable : 'DOCKERHUB_PASSWORD' ,usernameVariable : 'DOCKERHUB_USERNAME' ,credentialsId : "$DOCKERHUB_CREDENTIAL_ID" ,)]) {
sh 'docker login --username $DOCKERHUB_USERNAME --password $DOCKERHUB_PASSWORD $REGISTRY_DOCKER_HUB'

sh 'docker tag $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_EDGE_CLIENT_SERVER:$VERSION_PRODUCTION $REGISTRY_DOCKER_HUB/fedml/fedml-edge-client-server:latest'
sh 'docker push $REGISTRY_DOCKER_HUB/fedml/fedml-edge-client-server:latest'

sh 'docker tag $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_BACKEND:latest $REGISTRY_DOCKER_HUB/fedml/fedml-inference-backend:latest'
sh 'docker push $REGISTRY_DOCKER_HUB/fedml/$APP_NAME_MODEL_INFERENCE_BACKEND:latest'

sh 'docker tag $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_INGRESS:$VERSION_PRODUCTION $REGISTRY_DOCKER_HUB/fedml/fedml-inference-ingress:$VERSION_PRODUCTION'
sh 'docker push $REGISTRY_DOCKER_HUB/fedml/$APP_NAME_MODEL_INFERENCE_INGRESS:$VERSION_PRODUCTION'

sh 'docker tag $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_MASTER:$VERSION_PRODUCTION $REGISTRY_DOCKER_HUB/fedml/fedml-model-premise-master:$VERSION_PRODUCTION'
sh 'docker push $REGISTRY_DOCKER_HUB/fedml/$APP_NAME_PREMISE_MASTER:$VERSION_PRODUCTION'

sh 'docker tag $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_SLAVE:$VERSION_PRODUCTION $REGISTRY_DOCKER_HUB/fedml/fedml-model-premise-slave:$VERSION_PRODUCTION'
sh 'docker push $REGISTRY_DOCKER_HUB/fedml/$APP_NAME_PREMISE_SLAVE:$VERSION_PRODUCTION'
}
// withCredentials([usernamePassword(passwordVariable : 'AWS_IAM_ACCESS_ID' ,usernameVariable : 'AWS_ECR_USERNAME' ,credentialsId : "$AWS_ECR_CREDENTIAL_ID_PRODUCTION" ,)]) {
// withCredentials([usernamePassword(passwordVariable : 'AWS_IAM_ACCESS_KEY' ,usernameVariable : 'AWS_REGION' ,credentialsId : "$AWS_IAM_CREDENTIAL_ID_PRODUCTION" ,)]) {
// sh 'curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"'
// sh 'unzip -o awscliv2.zip'
// sh './aws/install'
// sh 'chmod a+x ./devops/scripts/set-aws-credentials.sh;./devops/scripts/set-aws-credentials.sh $AWS_IAM_ACCESS_ID $AWS_IAM_ACCESS_KEY $AWS_REGION'
// sh 'docker login --username $AWS_ECR_USERNAME --password $(/usr/local/bin/aws ecr-public get-login-password --region $AWS_REGION) $REGISTRY_PRODUCTION'
// }
// }
//
// sh 'docker push public.ecr.aws/x6k8q1x9/fedml-device-image:base'
// sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_DEVICE_IMAGE:$VERSION_PRODUCTION'
// sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_SERVER_AGENT:$VERSION_PRODUCTION'
// sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_CLIENT_AGENT:$VERSION_PRODUCTION'
// sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_EDGE_SERVER:$VERSION_PRODUCTION'
// sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_DOCS_SERVER:$VERSION_PRODUCTION'
//
// sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_BACKEND:latest'
// sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_INGRESS:$VERSION_PRODUCTION'
// sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_MASTER:$VERSION_PRODUCTION'
// sh 'docker push $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_SLAVE:$VERSION_PRODUCTION'
//
// withCredentials([usernamePassword(passwordVariable : 'DOCKERHUB_PASSWORD' ,usernameVariable : 'DOCKERHUB_USERNAME' ,credentialsId : "$DOCKERHUB_CREDENTIAL_ID" ,)]) {
// sh 'docker login --username $DOCKERHUB_USERNAME --password $DOCKERHUB_PASSWORD $REGISTRY_DOCKER_HUB'
//
// sh 'docker tag $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_EDGE_CLIENT_SERVER:$VERSION_PRODUCTION $REGISTRY_DOCKER_HUB/fedml/fedml-edge-client-server:latest'
// sh 'docker push $REGISTRY_DOCKER_HUB/fedml/fedml-edge-client-server:latest'
//
// sh 'docker tag $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_BACKEND:latest $REGISTRY_DOCKER_HUB/fedml/fedml-inference-backend:latest'
// sh 'docker push $REGISTRY_DOCKER_HUB/fedml/$APP_NAME_MODEL_INFERENCE_BACKEND:latest'
//
// sh 'docker tag $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_MODEL_INFERENCE_INGRESS:$VERSION_PRODUCTION $REGISTRY_DOCKER_HUB/fedml/fedml-inference-ingress:$VERSION_PRODUCTION'
// sh 'docker push $REGISTRY_DOCKER_HUB/fedml/$APP_NAME_MODEL_INFERENCE_INGRESS:$VERSION_PRODUCTION'
//
// sh 'docker tag $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_MASTER:$VERSION_PRODUCTION $REGISTRY_DOCKER_HUB/fedml/fedml-model-premise-master:$VERSION_PRODUCTION'
// sh 'docker push $REGISTRY_DOCKER_HUB/fedml/$APP_NAME_PREMISE_MASTER:$VERSION_PRODUCTION'
//
// sh 'docker tag $REGISTRY_PRODUCTION/$DOCKER_NAMESPACE$APP_NAME_PREMISE_SLAVE:$VERSION_PRODUCTION $REGISTRY_DOCKER_HUB/fedml/fedml-model-premise-slave:$VERSION_PRODUCTION'
// sh 'docker push $REGISTRY_DOCKER_HUB/fedml/$APP_NAME_PREMISE_SLAVE:$VERSION_PRODUCTION'
// }
}
}
}
Expand Down Expand Up @@ -484,16 +484,17 @@ pipeline {
credentialsId: env.PRODUCTION_KUBECONFIG_CREDENTIAL_ID,
variable: 'KUBECONFIG')
]) {
sh 'kubectl rollout restart -n $DEPLOY_NAMESPACE_PRODUCTION deployment/fedml-cloud-server'

sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=0 deployment/$DEPLOY_NAME'
sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=1 deployment/$DEPLOY_NAME'

sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=0 deployment/$DEPLOY_NAME_2'
sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=1 deployment/$DEPLOY_NAME_2'

sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=0 deployment/$DEPLOY_NAME_3'
sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=1 deployment/$DEPLOY_NAME_3'
sh 'echo "deploying..."'
// sh 'kubectl rollout restart -n $DEPLOY_NAMESPACE_PRODUCTION deployment/fedml-cloud-server'
//
// sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=0 deployment/$DEPLOY_NAME'
// sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=1 deployment/$DEPLOY_NAME'
//
// sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=0 deployment/$DEPLOY_NAME_2'
// sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=1 deployment/$DEPLOY_NAME_2'
//
// sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=0 deployment/$DEPLOY_NAME_3'
// sh 'kubectl scale -n $DEPLOY_NAMESPACE_PRODUCTION --replicas=1 deployment/$DEPLOY_NAME_3'
}

withCredentials([
Expand Down
2 changes: 1 addition & 1 deletion python/fedml/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
_global_training_type = None
_global_comm_backend = None

__version__ = "0.8.12.a1"
__version__ = "0.8.12.a2"


# This is the deployment environment used for different roles (RD/PM/BD/Public Developers). Potential VALUE: local, dev, test, release
Expand Down
4 changes: 4 additions & 0 deletions python/fedml/computing/scheduler/comm_utils/sys_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,10 @@ def versions(configuration_env, pkg_name):


def upgrade_if_not_latest():
disable_ota = os.getenv("FEDML_LAUNCH_DISABLE_OTA")
if disable_ota is not None and disable_ota == "YES":
return

try:
config_version = fedml.get_env_version()
is_latest_version, _, _ = check_fedml_is_latest_version()
Expand Down
2 changes: 1 addition & 1 deletion python/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ def finalize_options(self):

setup(
name="fedml",
version="0.8.12.a1",
version="0.8.12.a2",
author="FedML Team",
author_email="[email protected]",
description="A research and production integrated edge-cloud library for "
Expand Down

0 comments on commit e29ac57

Please sign in to comment.