add: refactor custom build & add disabled config options

Author: FabianKramm

examples/custom-builder/custom/build

@@ -9,15 +9,10 @@ images:
     image: ${IMAGE}
     build:
       custom:
-        command: ./custom/build
-        # command: docker
-        # args:
-        # - build
-        # - .
-        # - --tag
-        # appendArgs:
-        # - --file
-        # - ./custom/Dockerfile
+        skipImageArg: true
+        command: |-
+          # Also works on windows as this is executed in a golang shell
+          docker build -t ${runtime.images.default.image}:${runtime.images.default.tag} . -f custom/Dockerfile
         onChange:
         - main.go
 deployments:

examples/custom-builder/custom/build.cmd

@@ -4,6 +4,10 @@ metadata:
   name: devspace
 spec:
   replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: devspace-default
+      app.kubernetes.io/name: devspace-app
   template:
     metadata:
       labels:
@@ -12,4 +16,5 @@ spec:
     spec:
       containers:
         - name: default
-          image: devspace
+          image: myusername/devspace
+          imagePullPolicy: Never

examples/custom-builder/devspace.yaml

@@ -2,6 +2,10 @@ package custom
 
 import (
 	"fmt"
+	"github.com/loft-sh/devspace/pkg/devspace/config"
+	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable/runtime"
+	"github.com/loft-sh/devspace/pkg/devspace/dependency/types"
+	"github.com/loft-sh/devspace/pkg/util/shell"
 	"io"
 	"path/filepath"
 	"strings"
@@ -28,14 +32,20 @@ type Builder struct {
 
 	imageConfigName string
 	imageTags       []string
+
+	config       config.Config
+	dependencies []types.Dependency
 }
 
 // NewBuilder creates a new custom builder
-func NewBuilder(imageConfigName string, imageConf *latest.ImageConfig, imageTags []string) *Builder {
+func NewBuilder(imageConfigName string, imageConf *latest.ImageConfig, imageTags []string, config config.Config, dependencies []types.Dependency) *Builder {
 	return &Builder{
 		imageConfigName: imageConfigName,
 		imageConf:       imageConf,
 		imageTags:       imageTags,
+
+		config:       config,
+		dependencies: dependencies,
 	}
 }
 
@@ -74,7 +84,7 @@ func (b *Builder) ShouldRebuild(cache *generated.CacheConfig, forceRebuild bool,
 	imageCache := cache.GetImageCache(b.imageConfigName)
 
 	// only rebuild Docker image when Dockerfile or context has changed since latest build
-	mustRebuild := imageCache.Tag == "" || imageCache.ImageConfigHash != imageConfigHash || imageCache.CustomFilesHash != customFilesHash
+	mustRebuild := forceRebuild || b.imageConf.RebuildStrategy == latest.RebuildStrategyAlways || imageCache.Tag == "" || imageCache.ImageConfigHash != imageConfigHash || imageCache.CustomFilesHash != customFilesHash
 
 	imageCache.ImageConfigHash = imageConfigHash
 	imageCache.CustomFilesHash = customFilesHash
@@ -87,8 +97,23 @@ func (b *Builder) Build(devspacePID string, log logpkg.Logger) error {
 	// Build arguments
 	args := []string{}
 
-	// add args
-	args = append(args, b.imageConf.Build.Custom.Args...)
+	// resolve command
+	if len(b.imageTags) > 0 {
+		key := fmt.Sprintf("images.%s", b.imageConfigName)
+		b.config.SetRuntimeVariable(key, b.imageConf.Image+":"+b.imageTags[0])
+		b.config.SetRuntimeVariable(key+".image", b.imageConf.Image)
+		b.config.SetRuntimeVariable(key+".tag", b.imageTags[0])
+	}
+	
+	// loop over args
+	for i := range b.imageConf.Build.Custom.Args {
+		resolvedArg, err := runtime.NewRuntimeResolver(false).FillRuntimeVariablesAsString(b.imageConf.Build.Custom.Args[i], b.config, b.dependencies)
+		if err != nil {
+			return err
+		}
+		
+		args = append(args, resolvedArg)
+	}
 
 	// add image arg
 	if !b.imageConf.Build.Custom.SkipImageArg {
@@ -106,7 +131,14 @@ func (b *Builder) Build(devspacePID string, log logpkg.Logger) error {
 	}
 
 	// append the rest
-	args = append(args, b.imageConf.Build.Custom.AppendArgs...)
+	for i := range b.imageConf.Build.Custom.AppendArgs {
+		resolvedArg, err := runtime.NewRuntimeResolver(false).FillRuntimeVariablesAsString(b.imageConf.Build.Custom.AppendArgs[i], b.config, b.dependencies)
+		if err != nil {
+			return err
+		}
+
+		args = append(args, resolvedArg)
+	}
 
 	// get the command
 	commandPath := b.imageConf.Build.Custom.Command
@@ -122,11 +154,11 @@ func (b *Builder) Build(devspacePID string, log logpkg.Logger) error {
 		return fmt.Errorf("no command specified for custom builder")
 	}
 
-	// make sure the path has the correct slashes
-	commandPath = filepath.FromSlash(commandPath)
-
-	// Create the command
-	cmd := command.NewStreamCommand(commandPath, args)
+	// resolve command and args
+	commandPath, err := runtime.NewRuntimeResolver(false).FillRuntimeVariablesAsString(commandPath, b.config, b.dependencies)
+	if err != nil {
+		return err
+	}
 
 	// Determine output writer
 	var writer io.Writer
@@ -138,9 +170,16 @@ func (b *Builder) Build(devspacePID string, log logpkg.Logger) error {
 
 	log.Infof("Build %s:%s with custom command '%s %s'", b.imageConf.Image, b.imageTags[0], commandPath, strings.Join(args, " "))
 
-	err := cmd.Run(writer, writer, nil)
-	if err != nil {
-		return errors.Errorf("Error building image: %v", err)
+	if len(args) == 0 {
+		err = shell.ExecuteShellCommand(commandPath, args, filepath.Dir(b.config.Path()), writer, writer, nil)
+		if err != nil {
+			return errors.Errorf("error building image: %v", err)
+		}
+	} else {
+		err = command.NewStreamCommand(commandPath, args).Run(writer, writer, nil)
+		if err != nil {
+			return errors.Errorf("error building image: %v", err)
+		}
 	}
 
 	log.Done("Done processing image '" + b.imageConf.Image + "'")

examples/custom-builder/kube/deployment.yaml

@@ -24,7 +24,7 @@ func (c *controller) createBuilder(imageConfigName string, imageConf *latest.Ima
 	var builder builder.Interface
 
 	if imageConf.Build != nil && imageConf.Build.Custom != nil {
-		builder = custom.NewBuilder(imageConfigName, imageConf, imageTags)
+		builder = custom.NewBuilder(imageConfigName, imageConf, imageTags, c.config, c.dependencies)
 	} else if imageConf.Build != nil && imageConf.Build.BuildKit != nil {
 		log.StartWait("Creating BuildKit builder")
 		defer log.StopWait()

pkg/devspace/build/builder/custom/custom.go

@@ -2,6 +2,7 @@ package loader
 
 import (
 	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable"
+	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable/runtime"
 	"github.com/loft-sh/devspace/pkg/devspace/config/versions"
 	"github.com/loft-sh/devspace/pkg/devspace/config/versions/latest"
 	"github.com/loft-sh/devspace/pkg/util/log"
@@ -96,16 +97,7 @@ func (p *profilesParser) Parse(originalRawConfig map[interface{}]interface{}, ra
 
 func fillVariablesAndParse(resolver variable.Resolver, preparedConfig map[interface{}]interface{}, log log.Logger) (*latest.Config, error) {
 	// fill in variables and expressions (leave out
-	preparedConfigInterface, err := resolver.FillVariablesExclude(preparedConfig, []string{
-		"/deployments/*/helm/values/**",
-		"/hooks/*/command",
-		"/hooks/*/args/*",
-		"/hooks/*/container/imageSelector",
-		"/dev/ports/*/imageSelector",
-		"/dev/sync/*/imageSelector",
-		"/dev/logs/*/selectors/*/imageSelector",
-		"/dev/terminal/imageSelector",
-	})
+	preparedConfigInterface, err := resolver.FillVariablesExclude(preparedConfig, runtime.Locations)
 	if err != nil {
 		return nil, err
 	}

pkg/devspace/build/create_builder.go

@@ -3,6 +3,7 @@ package variable
 import (
 	"fmt"
 	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable/expression"
+	"github.com/loft-sh/devspace/pkg/devspace/config/loader/variable/runtime"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -66,7 +67,7 @@ func (r *resolver) fillVariables(haystack interface{}, exclude []*regexp.Regexp)
 				return nil, err
 			}
 		}
-		
+
 		return t, nil
 	}
 
@@ -120,6 +121,13 @@ func (r *resolver) FindVariables(haystack interface{}) (map[string]bool, error)
 		}
 	}
 
+	// filter out runtime environment variables
+	for k := range varsUsed {
+		if strings.HasPrefix(k, "runtime.") {
+			delete(varsUsed, k)
+		}
+	}
+
 	return varsUsed, nil
 }
 
@@ -396,7 +404,7 @@ func (r *resolver) fillVariable(name string, definition *latest.Variable) (inter
 
 	// is runtime variable
 	if strings.HasPrefix(name, "runtime.") {
-		return nil, fmt.Errorf("cannot resolve %s in this config area as this config region is loaded on startup. Please check the DevSpace docs in which config regions you can use runtime variables", name)
+		return nil, fmt.Errorf("cannot resolve %s in this config area as this config region is loaded on startup. You can only use runtime variables in the following locations: \n  %s", name, strings.Join(runtime.Locations, "\n  "))
 	}
 
 	// fill variable without definition

pkg/devspace/config/loader/parser.go

@@ -7,6 +7,21 @@ import (
 	"strings"
 )
 
+var Locations = []string{
+	"/images/*/build/custom/command",
+	"/images/*/build/custom/commands/*/command",
+	"/images/*/build/custom/args/*",
+	"/images/*/build/custom/appendArgs/*",
+	"/deployments/*/helm/values/**",
+	"/hooks/*/command",
+	"/hooks/*/args/*",
+	"/hooks/*/container/imageSelector",
+	"/dev/ports/*/imageSelector",
+	"/dev/sync/*/imageSelector",
+	"/dev/logs/*/selectors/*/imageSelector",
+	"/dev/terminal/imageSelector",
+}
+
 // NewRuntimeVariable creates a new variable that is loaded during runtime
 func NewRuntimeVariable(name string, config config.Config, dependencies []types.Dependency, builtImages map[string]string) *runtimeVariable {
 	return &runtimeVariable{
@@ -57,6 +72,12 @@ func (e *runtimeVariable) Load() (bool, interface{}, error) {
 		return false, nil, fmt.Errorf("couldn't find runtime variable %s", e.name)
 	}
 
+	// generic retrieve runtime variable
+	out, ok := runtimeVariables[runtimeVar]
+	if ok {
+		return false, out, nil
+	}
+
 	// get image info from generated config
 	if strings.HasPrefix(runtimeVar, "images.") {
 		runtimeVar = strings.TrimPrefix(runtimeVar, "images.")
@@ -126,11 +147,5 @@ func (e *runtimeVariable) Load() (bool, interface{}, error) {
 		return false, nil, fmt.Errorf("couldn't find imageName %s resolving variable %s", imageName, e.name)
 	}
 
-	// generic retrieve runtime variable
-	out, ok := runtimeVariables[runtimeVar]
-	if !ok {
-		return false, nil, fmt.Errorf("couldn't find runtime variable %s", e.name)
-	}
-
-	return false, out, nil
+	return false, nil, fmt.Errorf("couldn't find runtime variable %s", e.name)
 }

pkg/devspace/config/loader/variable/resolver.go

@@ -448,6 +448,7 @@ type BuildOptions struct {
 type DeploymentConfig struct {
 	Name      string         `yaml:"name" json:"name"`
 	Namespace string         `yaml:"namespace,omitempty" json:"namespace,omitempty"`
+	Disabled  bool           `yaml:"disabled,omitempty" json:"disabled,omitempty"`
 	Helm      *HelmConfig    `yaml:"helm,omitempty" json:"helm,omitempty"`
 	Kubectl   *KubectlConfig `yaml:"kubectl,omitempty" json:"kubectl,omitempty"`
 }
@@ -956,6 +957,7 @@ type PodPatch struct {
 type DependencyConfig struct {
 	Name                     string          `yaml:"name" json:"name"`
 	Source                   *SourceConfig   `yaml:"source" json:"source"`
+	Disabled                 bool            `yaml:"disabled,omitempty" json:"disabled,omitempty"`
 	Profile                  string          `yaml:"profile,omitempty" json:"profile,omitempty"`
 	Profiles                 []string        `yaml:"profiles,omitempty" json:"profiles,omitempty"`
 	ProfileParents           []string        `yaml:"profileParents,omitempty" json:"profileParents,omitempty"`
@@ -1012,6 +1014,9 @@ type HookConfig struct {
 	// Name is the name of the hook
 	Name string `yaml:"name,omitempty" json:"name,omitempty"`
 
+	// If true, the hook is disabled and not executed
+	Disabled bool `yaml:"disabled,omitempty" json:"disabled,omitempty"`
+
 	// Events are the events when the hook should be executed
 	Events []string `yaml:"events" json:"events"`
 
@@ -1212,6 +1217,9 @@ type PatchConfig struct {
 
 // PullSecretConfig defines a pull secret that should be created by DevSpace
 type PullSecretConfig struct {
+	// If true, the pull secret will be not created
+	Disabled bool `yaml:"disabled,omitempty" json:"disabled,omitempty"`
+
 	// The registry to create the image pull secret for.
 	// e.g. gcr.io
 	Registry string `yaml:"registry" json:"registry"`