Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
233 changes: 233 additions & 0 deletions backend/ci_backends/azure_devops.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
package ci_backends

import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
neturl "net/url"
"os"
"sync"

"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/diggerhq/digger/libs/spec"
)

var (
azureCred *azidentity.DefaultAzureCredential
azureCredOnce sync.Once
azureCredErr error
)

func getAzureCredential() (*azidentity.DefaultAzureCredential, error) {
azureCredOnce.Do(func() {
azureCred, azureCredErr = azidentity.NewDefaultAzureCredential(nil)
})
return azureCred, azureCredErr
}

type AzureDevOpsCi struct {
Client *http.Client
AuthToken string
IsBasicAuth bool
}

var (
pipelineIDCache = make(map[string]string)
pipelineIDCacheMu sync.Mutex
)

// resolvePipelineID resolves a workflow_file (pipeline name) to its numeric
// Azure DevOps pipeline ID by querying the Build Definitions API. Results are
// cached for the lifetime of the process. Falls back to AZURE_DEVOPS_PIPELINE_ID
// for simple single-pipeline deployments.
func (a AzureDevOpsCi) resolvePipelineID(pipelineName string) (string, error) {
pipelineIDCacheMu.Lock()
if id, ok := pipelineIDCache[pipelineName]; ok {
pipelineIDCacheMu.Unlock()
return id, nil
}
pipelineIDCacheMu.Unlock()

org := os.Getenv("AZURE_DEVOPS_ORG")
project := os.Getenv("AZURE_DEVOPS_PROJECT")
if org == "" || project == "" {
return "", fmt.Errorf("AZURE_DEVOPS_ORG and AZURE_DEVOPS_PROJECT must be set")
}

apiURL := fmt.Sprintf(
"https://dev.azure.com/%s/%s/_apis/build/definitions?name=%s&api-version=7.1",
neturl.PathEscape(org), neturl.PathEscape(project), neturl.QueryEscape(pipelineName),
)

req, err := http.NewRequest("GET", apiURL, nil)
if err != nil {
return "", fmt.Errorf("failed to create pipeline lookup request: %v", err)
}
if a.IsBasicAuth {
req.SetBasicAuth("", a.AuthToken)
} else {
req.Header.Set("Authorization", "Bearer "+a.AuthToken)
}

resp, err := a.Client.Do(req)
if err != nil {
return a.pipelineIDFallback(pipelineName, fmt.Errorf("API request failed: %v", err))
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return a.pipelineIDFallback(pipelineName, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body)))
}

var result struct {
Count int `json:"count"`
Value []struct {
ID int `json:"id"`
Name string `json:"name"`
} `json:"value"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return a.pipelineIDFallback(pipelineName, fmt.Errorf("failed to decode response: %v", err))
}

if result.Count == 0 {
return a.pipelineIDFallback(pipelineName, fmt.Errorf("no pipeline found with name %q", pipelineName))
}

id := fmt.Sprintf("%d", result.Value[0].ID)
slog.Info("Resolved ADO pipeline name to ID", "name", pipelineName, "id", id)

pipelineIDCacheMu.Lock()
pipelineIDCache[pipelineName] = id
pipelineIDCacheMu.Unlock()

return id, nil
}

// pipelineIDFallback returns AZURE_DEVOPS_PIPELINE_ID when the API-based
// lookup cannot be performed. If the env var is unset it surfaces the
// original error.
func (a AzureDevOpsCi) pipelineIDFallback(pipelineName string, lookupErr error) (string, error) {
if fallback := os.Getenv("AZURE_DEVOPS_PIPELINE_ID"); fallback != "" {
slog.Warn("Pipeline name lookup failed, falling back to AZURE_DEVOPS_PIPELINE_ID",
"name", pipelineName, "error", lookupErr)
return fallback, nil
}
return "", fmt.Errorf("failed to resolve pipeline %q: %v", pipelineName, lookupErr)
}

func NewAzureDevOpsCi() (*AzureDevOpsCi, error) {
pat := os.Getenv("AZURE_DEVOPS_PAT")

ci := &AzureDevOpsCi{
Client: &http.Client{},
}

if pat != "" {
ci.AuthToken = pat
ci.IsBasicAuth = true
} else {
cred, err := getAzureCredential()
if err != nil {
return nil, fmt.Errorf("failed to obtain Azure credential: %v", err)
}
token, err := cred.GetToken(context.Background(), policy.TokenRequestOptions{
Scopes: []string{"499b84ac-1181-4cfa-b6f4-a0fa2c36415b/.default"},
})
if err != nil {
return nil, fmt.Errorf("failed to get Azure DevOps token: %v", err)
}
ci.AuthToken = token.Token
}

return ci, nil
}

func (a AzureDevOpsCi) TriggerWorkflow(spec spec.Spec, runName string, vcsToken string) error {
slog.Info("TriggerAzureDevOpsWorkflow", "repoOwner", spec.VCS.RepoOwner, "repoName", spec.VCS.RepoName, "commentId", spec.CommentId)

org := os.Getenv("AZURE_DEVOPS_ORG")
project := os.Getenv("AZURE_DEVOPS_PROJECT")

if org == "" || project == "" {
return fmt.Errorf("AZURE_DEVOPS_ORG and AZURE_DEVOPS_PROJECT must be set")
}

pipelineId, err := a.resolvePipelineID(spec.VCS.WorkflowFile)
if err != nil {
return fmt.Errorf("failed to resolve pipeline ID: %v", err)
}

specBytes, err := json.Marshal(spec)
if err != nil {
return fmt.Errorf("failed to marshal spec: %v", err)
}

payload := map[string]interface{}{
"resources": map[string]interface{}{
"repositories": map[string]interface{}{
"self": map[string]string{
"refName": "refs/heads/" + spec.Job.Branch,
},
},
},
"templateParameters": map[string]string{
"DIGGER_SPEC": string(specBytes),
"DIGGER_RUN_NAME": runName,
},
}

payloadBytes, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("failed to marshal payload: %v", err)
}

url := fmt.Sprintf("https://dev.azure.com/%s/%s/_apis/pipelines/%s/runs?api-version=7.1-preview.1", org, project, pipelineId)
req, err := http.NewRequest("POST", url, bytes.NewBuffer(payloadBytes))
if err != nil {
return fmt.Errorf("failed to create request: %v", err)
}
req.Header.Set("Content-Type", "application/json")

if a.IsBasicAuth {
req.SetBasicAuth("", a.AuthToken)
} else {
req.Header.Set("Authorization", "Bearer "+a.AuthToken)
}

resp, err := a.Client.Do(req)
if err != nil {
return fmt.Errorf("failed to send request to Azure DevOps: %v", err)
}
defer resp.Body.Close()

if resp.StatusCode < 200 || resp.StatusCode >= 300 {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("Azure DevOps API returned status %d: %s", resp.StatusCode, string(body))
}

return nil
}

func (a AzureDevOpsCi) GetWorkflowUrl(spec spec.Spec) (string, error) {
org := os.Getenv("AZURE_DEVOPS_ORG")
project := os.Getenv("AZURE_DEVOPS_PROJECT")

if org == "" || project == "" {
return "", fmt.Errorf("AZURE_DEVOPS_ORG and AZURE_DEVOPS_PROJECT must be set")
}

pipelineId, err := a.resolvePipelineID(spec.VCS.WorkflowFile)
if err != nil {
return "", fmt.Errorf("failed to resolve pipeline ID: %v", err)
}

// Returns the pipeline overview page as a fallback since the exact run ID isn't persisted locally
return fmt.Sprintf("https://dev.azure.com/%s/%s/_build?definitionId=%s", org, project, pipelineId), nil
}
12 changes: 12 additions & 0 deletions backend/ci_backends/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package ci_backends
import (
"fmt"
"log/slog"
"os"

"github.com/diggerhq/digger/backend/utils"
)
Expand All @@ -14,6 +15,17 @@ type CiBackendProvider interface {
type DefaultBackendProvider struct{}

func (d DefaultBackendProvider) GetCiBackend(options CiBackendOptions) (CiBackend, error) {
backendType := os.Getenv("DIGGER_CI_BACKEND")
if backendType == "azure_devops" {
slog.Info("Using Azure DevOps CI backend")
ci, err := NewAzureDevOpsCi()
if err != nil {
slog.Error("GetCiBackend: could not initialize Azure DevOps client", "error", err)
return nil, fmt.Errorf("could not initialize Azure DevOps client: %v", err)
}
return ci, nil
}

client, _, err := utils.GetGithubClientFromAppId(options.GithubClientProvider, options.GithubInstallationId, options.GithubAppId, options.RepoFullName)
if err != nil {
slog.Error("GetCiBackend: could not get github client", "error", err)
Expand Down
4 changes: 2 additions & 2 deletions backend/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ require (
filippo.io/age v1.2.1 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/Azure/azure-sdk-for-go v63.3.0+incompatible // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 // indirect
Expand Down
7 changes: 0 additions & 7 deletions backend/go.sum
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
ariga.io/atlas-go-sdk v0.7.2 h1:pvS8tKVeRQuqdETBqj5qAQtVbQE88Gya6bOfY8YF3vU=
ariga.io/atlas-go-sdk v0.7.2/go.mod h1:cFq7bnvHgKTWHCsU46mtkGxdl41rx2o7SjaLoh6cO8M=
ariga.io/atlas-provider-gorm v0.5.0 h1:DqYNWroKUiXmx2N6nf/I9lIWu6fpgB6OQx/JoelCTes=
ariga.io/atlas-provider-gorm v0.5.0/go.mod h1:8m6+N6+IgWMzPcR63c9sNOBoxfNk6yV6txBZBrgLg1o=
ariga.io/atlas-provider-gorm v0.5.4 h1:64xboUDrP+JHdZOy4juPydHT5UP1kY152b5Gh/xNzmM=
ariga.io/atlas-provider-gorm v0.5.4/go.mod h1:cXt4kxq8KIldPXHoWXC0HvSr8dVI0dIykZt3MZ4AmqE=
c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805 h1:u2qwJeEvnypw+OCPUHmoZE3IqwfuN5kgDfo5MLzpNM0=
Expand Down Expand Up @@ -759,10 +757,6 @@ github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=
github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
github.com/alecthomas/kong v1.9.0 h1:Wgg0ll5Ys7xDnpgYBuBn/wPeLGAuK0NvYmEcisJgrIs=
github.com/alecthomas/kong v1.9.0/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
Expand Down Expand Up @@ -2847,7 +2841,6 @@ gorm.io/gorm v1.23.7/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.23.10/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
gorm.io/gorm v1.30.1 h1:lSHg33jJTBxs2mgJRfRZeLDG+WZaHYCk3Wtfl6Ngzo4=
gorm.io/gorm v1.30.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
Expand Down
93 changes: 93 additions & 0 deletions docs/ce/cloud-providers/azure-devops-backend.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
---
title: "Azure DevOps Pipelines Backend"
description: "Configure Digger Orchestrator to use Azure DevOps Pipelines as the execution engine using Workload Identity."
---

# Azure DevOps Pipelines Backend

While Digger officially provides a standalone "backendless" integration for Azure DevOps, you can also configure the **Digger Orchestrator (Backend)** to use Azure DevOps Pipelines as the compute layer. This approach allows you to retain Digger's native OPA policy enforcement, central run history UI, and dependency-aware job batching while running Terraform execution within your existing Azure DevOps agents.

By natively supporting Azure AD Workload Identity (`azidentity`), Digger can securely trigger Azure Pipelines using the Managed Identity of the cluster hosting the Orchestrator (e.g., Azure Kubernetes Service).

## 1. Prerequisites

- A Digger Orchestrator deployment (running on a platform like AKS or GKE).
- A GitHub App configured and linked to the Digger Orchestrator.
- An Azure DevOps organization and project.
- An Azure Pipeline (YAML) defined in your target repository (or centrally) to receive Digger jobs.

## 2. Configure the Digger Orchestrator

Set the following environment variables on the Digger Orchestrator deployment to route jobs to Azure DevOps instead of GitHub Actions:

```env
DIGGER_CI_BACKEND=azure_devops
AZURE_DEVOPS_ORG=your-azure-devops-organization
AZURE_DEVOPS_PROJECT=your-azure-devops-project
AZURE_DEVOPS_PIPELINE_ID=123 # The ID of the pipeline definition to trigger
```

### Authentication
The Orchestrator requires permissions to trigger pipeline runs in Azure DevOps via the REST API. It uses the standard Azure SDK for Go credential chain.

**Option A: Workload Identity (Recommended)**
If the Orchestrator is running on Azure Kubernetes Service (AKS) with Workload Identity enabled, or on a VM with a Managed Identity, you do not need to provide any explicit credentials. Ensure the underlying identity is granted the **"Queue builds"** permission in the target Azure DevOps project.

**Option B: Personal Access Token (Fallback)**
If Workload Identity is unavailable or you are testing locally, you can provide an Azure DevOps Personal Access Token (PAT):
```env
AZURE_DEVOPS_PAT=your_azure_devops_pat
```

## 3. Configure the Azure Pipeline

When the Digger Orchestrator triggers an Azure Pipeline, it passes a JSON string containing the job specifications (commands, environment variables, PR details) as a template parameter named `DIGGER_SPEC`.

It also passes `DIGGER_RUN_NAME` to uniquely identify the run context.

Create an `azure-pipelines.yml` file in your repository:

```yaml
trigger: none # Digger will trigger this pipeline via the API

parameters:
- name: DIGGER_SPEC
type: string
default: ''
- name: DIGGER_RUN_NAME
type: string
default: 'digger-run'

jobs:
- job: DiggerExecution
displayName: 'Digger Terraform Execution'
pool:
name: 'Your-Self-Hosted-Agent-Pool' # e.g., the pool with your target Workload Identity
steps:
- script: |
echo "Processing Digger Job: $(DIGGER_RUN_NAME)"

# Download the Digger CLI
curl -sL -o digger https://github.com/diggerhq/digger/releases/latest/download/digger-cli-linux-amd64
chmod +x digger

# The Digger CLI expects the spec JSON to be available as an environment variable
export DIGGER_JOB_SPEC='${{ parameters.DIGGER_SPEC }}'

# Execute Digger
./digger run_spec
displayName: 'Run Digger CLI'
env:
# Ensure any specific variables required by your Terraform provider are passed here,
# or rely on the agent's native workload identity.
ARM_USE_OIDC: "true"
```

## 4. How It Works

1. A developer creates a Pull Request in GitHub.
2. The GitHub App sends a webhook to the Digger Orchestrator.
3. The Orchestrator calculates the impacted projects, evaluates OPA policies, and determines the necessary Terraform commands (e.g., `digger plan`).
4. Instead of triggering a GitHub Action, the Orchestrator uses its Managed Identity to authenticate to the Azure DevOps REST API.
5. It queues a new run of the specified `AZURE_DEVOPS_PIPELINE_ID`, passing the job specifications into the `DIGGER_SPEC` parameter.
6. The Azure Pipeline agent executes the `digger run_spec` command, performs the Terraform plan/apply using its own Workload Identity, and reports the status back to the Orchestrator.