primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 12:10:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add aad auth option when using azure storage account (#1973)

Browse Source 
Support managed identity authentication in Azure Blob Storage.
This commit is contained in:
yueluhuan
2024-07-15 09:21:31 +08:00
committed by GitHub
parent a1e728e5be
commit 1e39c23d72
9 changed files with 86 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.dev.toml
+8
View File
@@ -536,6 +536,14 @@ ShutdownTimeout = 60
# Env override: ATHENS_AZURE_ACCOUNT_KEY
AccountKey = "MY_AZURE_BLOB_ACCOUNT_KEY"
# Managed Identity Resource Id to use with the storage account
# Env override: ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID
ManagedIdentityResourceId = "MY_AZURE_MANAGED_IDENTITY_RESOURCE_ID"
# Storage Resource to use with the storage account
# Env override: ATHENS_AZURE_STORAGE_RESOURCE
StorageResource = "MY_AZURE_STORAGE_RESOURCE"
# Name of container in the blob storage
# Env override: ATHENS_AZURE_CONTAINER_NAME
ContainerName = "MY_AZURE_BLOB_CONTAINER_NAME"
docs/content/configuration/storage.md
+8
View File
@@ -339,6 +339,14 @@ It assumes that you already have the following:
# Env override: ATHENS_AZURE_ACCOUNT_KEY
AccountKey = "MY_AZURE_BLOB_ACCOUNT_KEY"
# Managed Identity Resource Id to use with the storage account
# Env override: ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID
ManagedIdentityResourceId = "MY_AZURE_MANAGED_IDENTITY_RESOURCE_ID"
# Storage Resource to use with the storage account
# Env override: ATHENS_AZURE_STORAGE_RESOURCE
StorageResource = "MY_AZURE_STORAGE_RESOURCE"
# Name of container in the blob storage
# Env override: ATHENS_AZURE_CONTAINER_NAME
ContainerName = "MY_AZURE_BLOB_CONTAINER_NAME"
go.mod
+5
View File
@@ -8,6 +8,7 @@ require (
contrib.go.opencensus.io/exporter/prometheus v0.1.0
contrib.go.opencensus.io/exporter/stackdriver v0.6.0
github.com/Azure/azure-storage-blob-go v0.10.0
github.com/Azure/go-autorest/autorest/adal v0.9.23
github.com/BurntSushi/toml v1.0.0
github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57
github.com/aws/aws-sdk-go-v2 v1.26.1
@@ -56,6 +57,10 @@ require (
cloud.google.com/go/monitoring v1.18.0 // indirect
cloud.google.com/go/trace v1.10.5 // indirect
github.com/Azure/azure-pipeline-go v0.2.2 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 // indirect
github.com/agext/levenshtein v1.2.1 // indirect
github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f // indirect
go.sum
+18 -5
View File
@@ -60,21 +60,29 @@ github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot
github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs=
github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE=
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs=
github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
github.com/Azure/go-autorest/autorest/adal v0.8.3 h1:O1AGG9Xig71FxdX9HO5pGNyZ7TbSyHaVg+5eJO/jSGw=
github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8=
github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c=
github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM=
github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY=
github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=
github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
@@ -160,7 +168,6 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
@@ -509,6 +516,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245 h1:DNVk+NIkGS0RbLkjQOLCJb/759yfCysThkMbl7EXxyY=
@@ -610,6 +618,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -691,6 +700,7 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -773,11 +783,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -789,6 +801,7 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
pkg/config/azureblob.go
+3 -1
View File
@@ -3,6 +3,8 @@ package config
// AzureBlobConfig specifies the properties required to use Azure as the storage backend.
type AzureBlobConfig struct {
AccountName string `envconfig:"ATHENS_AZURE_ACCOUNT_NAME" validate:"required"`
AccountKey string `envconfig:"ATHENS_AZURE_ACCOUNT_KEY" validate:"required"`
AccountKey string `envconfig:"ATHENS_AZURE_ACCOUNT_KEY"`
ManagedIdentityResourceID string `envconfig:"ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID"`
StorageResource string `envconfig:"ATHENS_AZURE_STORAGE_RESOURCE"`
ContainerName string `envconfig:"ATHENS_AZURE_CONTAINER_NAME" validate:"required"`
}
pkg/config/config_test.go
+2
View File
@@ -237,6 +237,8 @@ func TestParseExampleConfig(t *testing.T) {
AzureBlob: &AzureBlobConfig{
AccountName: "MY_AZURE_BLOB_ACCOUNT_NAME",
AccountKey: "MY_AZURE_BLOB_ACCOUNT_KEY",
ManagedIdentityResourceID: "MY_AZURE_MANAGED_IDENTITY_RESOURCE_ID",
StorageResource: "MY_AZURE_STORAGE_RESOURCE",
ContainerName: "MY_AZURE_BLOB_CONTAINER_NAME",
},
External: &External{URL: ""},
pkg/stash/with_azureblob.go
+19 -1
View File
@@ -9,6 +9,7 @@ import (
"time"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/Azure/go-autorest/autorest/adal"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
@@ -21,14 +22,31 @@ import (
func WithAzureBlobLock(conf *config.AzureBlobConfig, timeout time.Duration, checker storage.Checker) (Wrapper, error) {
const op errors.Op = "stash.WithAzureBlobLock"
if conf.AccountKey == "" && (conf.ManagedIdentityResourceID == "" || conf.StorageResource == "") {
return nil, errors.E(op, "either account key or managed identity resource id and storage resource must be set")
}
accountURL, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName))
if err != nil {
return nil, errors.E(op, err)
}
cred, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey)
var cred azblob.Credential
if conf.AccountKey != "" {
cred, err = azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey)
if err != nil {
return nil, errors.E(op, err)
}
}
if conf.ManagedIdentityResourceID != "" {
spStorageToken, err := adal.NewServicePrincipalTokenFromManagedIdentity(conf.StorageResource, &adal.ManagedIdentityOptions{IdentityResourceID: conf.ManagedIdentityResourceID})
if err != nil {
return nil, errors.E(op, err)
}
err = spStorageToken.Refresh()
if err != nil {
return nil, errors.E(op, err)
}
cred = azblob.NewTokenCredential(spStorageToken.OAuthToken(), nil)
}
pipe := azblob.NewPipeline(cred, azblob.PipelineOptions{})
serviceURL := azblob.NewServiceURL(*accountURL, pipe)
pkg/stash/with_azureblob_test.go
+5 -1
View File
@@ -86,7 +86,9 @@ func (ms *mockAzureBlobStasher) Stash(ctx context.Context, mod, ver string) (str
func getAzureTestConfig(containerName string) *config.AzureBlobConfig {
key := os.Getenv("ATHENS_AZURE_ACCOUNT_KEY")
if key == "" {
resourceId := os.Getenv("ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID")
storageResource := os.Getenv("ATHENS_AZURE_STORAGE_RESOURCE")
if key == "" && (resourceId == "" || storageResource == "") {
return nil
}
name := os.Getenv("ATHENS_AZURE_ACCOUNT_NAME")
@@ -96,6 +98,8 @@ func getAzureTestConfig(containerName string) *config.AzureBlobConfig {
return &config.AzureBlobConfig{
AccountName: name,
AccountKey: key,
ManagedIdentityResourceID: resourceId,
StorageResource: storageResource,
ContainerName: containerName,
}
}
pkg/storage/azureblob/azureblob_test.go
+5 -1
View File
@@ -70,7 +70,9 @@ func getStorage(t testing.TB) *Storage {
func getTestConfig(containerName string) *config.AzureBlobConfig {
key := os.Getenv("ATHENS_AZURE_ACCOUNT_KEY")
if key == "" {
resourceId := os.Getenv("ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID")
storageResource := os.Getenv("ATHENS_AZURE_STORAGE_RESOURCE")
if key == "" && (resourceId == "" || storageResource == "") {
return nil
}
name := os.Getenv("ATHENS_AZURE_ACCOUNT_NAME")
@@ -80,6 +82,8 @@ func getTestConfig(containerName string) *config.AzureBlobConfig {
return &config.AzureBlobConfig{
AccountName: name,
AccountKey: key,
ManagedIdentityResourceID: resourceId,
StorageResource: storageResource,
ContainerName: containerName,
}
}