Add digitialocean provider

cmd/woodpecker-autoscaler/main.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/woodpecker-ci/autoscaler/config"
 	"github.com/woodpecker-ci/autoscaler/engine"
+	"github.com/woodpecker-ci/autoscaler/providers/digitalocean"
 	"github.com/woodpecker-ci/autoscaler/providers/hetznercloud"
 	"github.com/woodpecker-ci/autoscaler/server"
 )
@@ -21,6 +22,8 @@ func setupProvider(ctx *cli.Context, config *config.Config) (engine.Provider, er
 	switch ctx.String("provider") {
 	case "hetznercloud":
 		return hetznercloud.New(ctx, config)
+	case "digitalocean":
+		return digitalocean.New(ctx, config)
 	case "":
 		return nil, fmt.Errorf("Please select a provider")
 	}

engine/stringmaps.go

@@ -19,6 +19,14 @@ func SliceToMap(list []string, del string) (map[string]string, error) {
 	return m, nil
 }
 
+func MapToSlice(m map[string]string, del string) []string {
+	var list []string
+	for k, v := range m {
+		list = append(list, fmt.Sprintf("%s%s%s", k, del, v))
+	}
+	return list
+}
+
 func MergeMaps(m1, m2 map[string]string) map[string]string {
 	merged := make(map[string]string)
 	for k, v := range m1 {

go.mod

@@ -3,6 +3,7 @@ module github.com/woodpecker-ci/autoscaler
 go 1.20
 
 require (
+	github.com/digitalocean/godo v1.102.1
 	github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf
 	github.com/hetznercloud/hcloud-go/v2 v2.1.0
 	github.com/joho/godotenv v1.5.1
@@ -19,6 +20,9 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
@@ -30,6 +34,7 @@ require (
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/sys v0.10.0 // indirect
 	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 )

go.sum

@@ -6,6 +6,9 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/digitalocean/godo v1.102.1 h1:BrNePwIXjQWjOJXVTBqkURMjm70BRR0qXbRKfHNBF24=
+github.com/digitalocean/godo v1.102.1/go.mod h1:SaUYccN7r+CO1QtsbXGypAsgobDrmSfVMJESEfXgoEg=
 github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf h1:NrF81UtW8gG2LBGkXFQFqlfNnvMt9WdB46sfdJY4oqc=
 github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -15,8 +18,17 @@ github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgj
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
+github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hetznercloud/hcloud-go/v2 v2.1.0 h1:NJEQ9pdF8qerYav7ZH/tpzC/1+Q4UR7vIq/s/CmJ4Jk=
 github.com/hetznercloud/hcloud-go/v2 v2.1.0/go.mod h1:4iUG2NG8b61IAwNx6UsMWQ6IfIf/i1RsG0BbsKAyR5Q=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
@@ -32,6 +44,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
 github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
 github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
@@ -45,6 +58,7 @@ github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
 github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/urfave/cli/v2 v2.25.3 h1:VJkt6wvEBOoSjPFQvOkv6iWIrsJyCrKGtCtxXWwmGeY=
 github.com/urfave/cli/v2 v2.25.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
@@ -71,6 +85,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
 golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=

providers/digitalocean/flags.go

@@ -0,0 +1,73 @@
+package digitalocean
+
+import (
+	"os"
+
+	"github.com/urfave/cli/v2"
+)
+
+const category = "Digtial Ocean"
+
+var DriverFlags = []cli.Flag{
+	// digitalocean
+	&cli.StringFlag{
+		Name:     "digitalocean-api-token",
+		Usage:    "digitalocean api token",
+		EnvVars:  []string{"WOODPECKER_DIGITALOCEAN_API_TOKEN"},
+		FilePath: os.Getenv("WOODPECKER_DIGITALOCEAN_API_TOKEN_FILE"),
+		Category: category,
+	},
+	&cli.StringFlag{
+		Name:     "digitalocean-region",
+		Value:    "nbg1", // TODO
+		Usage:    "digitalocean region",
+		EnvVars:  []string{"WOODPECKER_DIGITALOCEAN_REGION"},
+		Category: category,
+	},
+	&cli.StringFlag{
+		Name:     "digitalocean-droplet-size",
+		Value:    "cx11",
+		Usage:    "digitalocean server type",
+		EnvVars:  []string{"WOODPECKER_DIGITALOCEAN_SERVER_TYPE"},
+		Category: category,
+	},
+	&cli.StringSliceFlag{
+		Name:     "digitalocean-ssh-keys",
+		Usage:    "names of digitalocean ssh keys",
+		EnvVars:  []string{"WOODPECKER_DIGITALOCEAN_SSH_KEYS"},
+		Category: category,
+	},
+	&cli.StringFlag{
+		Name:     "digitalocean-user-data",
+		Usage:    "digitalocean userdata template",
+		EnvVars:  []string{"WOODPECKER_DIGITALOCEAN_USERDATA"},
+		FilePath: os.Getenv("WOODPECKER_DIGITALOCEAN_USERDATA_FILE"),
+		Category: category,
+	},
+	&cli.StringFlag{
+		Name:     "digitalocean-image",
+		Value:    "ubuntu-22.04",
+		Usage:    "digitalocean image",
+		EnvVars:  []string{"WOODPECKER_DIGITALOCEAN_IMAGE"},
+		Category: category,
+	},
+	&cli.StringSliceFlag{
+		Name:     "digitalocean-labels",
+		Usage:    "digitalocean server labels",
+		EnvVars:  []string{"WOODPECKER_DIGITALOCEAN_LABELS"},
+		Category: category,
+	},
+	&cli.StringSliceFlag{
+		Name:     "digitalocean-firewall",
+		Usage:    "names of digitalocean firewall",
+		EnvVars:  []string{"WOODPECKER_DIGITALOCEAN_FIREWALL"},
+		Category: category,
+	},
+	&cli.BoolFlag{
+		Name:     "digitalocean-public-ipv6-enable",
+		Value:    true,
+		Usage:    "enables public ipv6 network for agents",
+		EnvVars:  []string{"WOODPECKER_DIGITALOCEAN_PUBLIC_IPV6_ENABLE"},
+		Category: category,
+	},
+}

providers/digitalocean/provider.go

@@ -0,0 +1,139 @@
+package digitalocean
+
+import (
+	"context"
+	"fmt"
+	"text/template"
+
+	"github.com/digitalocean/godo"
+	"github.com/urfave/cli/v2"
+
+	"github.com/woodpecker-ci/autoscaler/config"
+	"github.com/woodpecker-ci/autoscaler/engine"
+	"github.com/woodpecker-ci/woodpecker/woodpecker-go/woodpecker"
+)
+
+var (
+// ErrIllegalLablePrefix = errors.New("illegal label prefix")
+// ErrImageNotFound      = errors.New("image not found")
+// ErrSSHKeyNotFound     = errors.New("SSH key not found")
+// ErrNetworkNotFound    = errors.New("network not found")
+// ErrFirewallNotFound   = errors.New("firewall not found")
+)
+
+type Provider struct {
+	name        string
+	dropletSize string
+	userData    *template.Template
+	image       string
+	sshKeys     []string
+	labels      map[string]string
+	config      *config.Config
+	region      string
+	firewall    string
+	enableIPv6  bool
+	client      *godo.Client
+}
+
+func New(c *cli.Context, config *config.Config) (engine.Provider, error) {
+	d := &Provider{
+		name:        "digitalocean",
+		region:      c.String("digitalocean-region"),
+		dropletSize: c.String("digitalocean-droplet-size"),
+		image:       c.String("hetznercloud-image"),
+		sshKeys:     c.StringSlice("hetznercloud-ssh-keys"),
+		enableIPv6:  c.Bool("hetznercloud-public-ipv6-enable"),
+		config:      config,
+	}
+
+	d.client = godo.NewFromToken(c.String("digitalocean-api-token"))
+
+	return d, nil
+}
+
+func (d *Provider) DeployAgent(ctx context.Context, agent *woodpecker.Agent) error {
+	userdataString, err := engine.RenderUserDataTemplate(d.config, agent, d.userData)
+	if err != nil {
+		return fmt.Errorf("%s: RenderUserDataTemplate: %w", d.name, err)
+	}
+
+	droplet, _, err := d.client.Droplets.Create(ctx, &godo.DropletCreateRequest{
+		Name:     agent.Name,
+		UserData: userdataString,
+		Image: godo.DropletCreateImage{
+			Slug: d.image,
+		},
+		Region:  d.region,
+		Size:    d.dropletSize,
+		IPv6:    d.enableIPv6,
+		Backups: false,
+		Tags:    engine.MapToSlice(d.labels, "="),
+		SSHKeys: []godo.DropletCreateSSHKey{
+			{Fingerprint: d.sshKeys[0]}, // TODO: support multiple SSH keys
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("%s: Droplets.Create: %w", d.name, err)
+	}
+
+	// TODO: support firewalls
+	if d.firewall != "" {
+		_, err := d.client.Firewalls.AddDroplets(ctx, d.firewall, droplet.ID)
+		if err != nil {
+			return fmt.Errorf("%s: Firewalls.AddDroplets: %w", d.name, err)
+		}
+	}
+
+	return nil
+}
+
+func (d *Provider) getDroplet(ctx context.Context, agent *woodpecker.Agent) (*godo.Droplet, error) {
+	droplet, _, err := d.client.Droplets.ListByName(ctx, agent.Name, &godo.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", d.name, err)
+	}
+
+	if len(droplet) == 0 {
+		return nil, nil
+	}
+
+	if len(droplet) > 1 {
+		return nil, fmt.Errorf("%s: getDroplet: multiple droplets found for %s", d.name, agent.Name)
+	}
+
+	return &droplet[0], nil
+}
+
+func (d *Provider) RemoveAgent(ctx context.Context, agent *woodpecker.Agent) error {
+	droplet, err := d.getDroplet(ctx, agent)
+	if err != nil {
+		return fmt.Errorf("%s: getDroplet %w", d.name, err)
+	}
+
+	if droplet == nil {
+		return nil
+	}
+
+	_, err = d.client.Droplets.Delete(ctx, droplet.ID)
+	if err != nil {
+		return fmt.Errorf("%s: Droplets.Delete %w", d.name, err)
+	}
+
+	return nil
+}
+
+func (d *Provider) ListDeployedAgentNames(ctx context.Context) ([]string, error) {
+	var names []string
+
+	droplets, _, err := d.client.Droplets.ListByTag(ctx,
+		fmt.Sprintf("%s=%s", engine.LabelPool, d.config.PoolID), &godo.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", d.name, err)
+	}
+
+	for _, droplet := range droplets {
+		names = append(names, droplet.Name)
+	}
+
+	return names, nil
+}