2023-07-05 08:51:03 +02:00
/ *
Copyright 2020 Docker Compose CLI authors
Licensed under the Apache License , Version 2.0 ( the "License" ) ;
you may not use this file except in compliance with the License .
You may obtain a copy of the License at
http : //www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing , software
distributed under the License is distributed on an "AS IS" BASIS ,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
See the License for the specific language governing permissions and
limitations under the License .
* /
package compose
import (
2025-03-10 17:26:07 +01:00
"bytes"
2023-07-05 08:51:03 +02:00
"context"
2025-03-05 11:41:43 +01:00
"crypto/sha256"
2025-03-03 14:51:39 +01:00
"errors"
2025-01-21 09:51:56 +01:00
"fmt"
2025-03-10 17:26:07 +01:00
"io"
2023-09-15 19:48:21 +02:00
"os"
2023-07-05 08:51:03 +02:00
2025-03-10 17:26:07 +01:00
"github.com/DefangLabs/secret-detector/pkg/scanner"
"github.com/DefangLabs/secret-detector/pkg/secrets"
2025-03-05 11:41:43 +01:00
"github.com/compose-spec/compose-go/v2/loader"
2023-11-08 10:19:24 +01:00
"github.com/compose-spec/compose-go/v2/types"
2023-08-30 10:15:35 +02:00
"github.com/distribution/reference"
2023-09-15 19:48:21 +02:00
"github.com/docker/buildx/util/imagetools"
2025-01-21 17:50:11 +01:00
"github.com/docker/cli/cli/command"
2023-12-04 22:39:23 -05:00
"github.com/docker/compose/v2/internal/ocipush"
2023-07-05 08:51:03 +02:00
"github.com/docker/compose/v2/pkg/api"
2025-03-07 17:52:16 +01:00
"github.com/docker/compose/v2/pkg/compose/transform"
2023-09-15 19:48:21 +02:00
"github.com/docker/compose/v2/pkg/progress"
2025-01-21 17:50:11 +01:00
"github.com/docker/compose/v2/pkg/prompt"
fix(publish): add OCI 1.0 fallback support for AWS ECR
Currently, we publish Compose artifacts following the OCI 1.1
specification, which is still in the RC state.
As a result, not all registries support it yet. Most notably,
AWS ECR will reject certain OCI 1.1-compliant requests with
`405 Method Not Supported` with cryptic `Invalid JSON` errors.
This adds initial support for Compose to generate either an
OCI 1.0 or OCI 1.1 compatible manifest. Notably, the OCI 1.0
manifest will be missing the `application/vnd.docker.compose.project`
artifact type, as that does not exist in that version of the
spec. (Less importantly, it uses an empty `ImageConfig`
instead of the newer `application/vnd.oci.empty.v1+json` media
type for the config.)
Currently, this is not exposed as an option (via CLI flags or
env vars). By default, OCI 1.1 is used unless the registry
domain is `amazonaws.com`, which indicates an ECR registry, so
Compose will instead use OCI 1.0.
Moving forward, we should decide how much we want to expose/
support different OCI versions and investigate if there's a
more generic way to feature probe the registry to avoid
maintaining a hardcoded list of domains, which is both tedious
and insufficient.
Signed-off-by: Milas Bowman <milas.bowman@docker.com>
2023-12-01 13:33:00 -05:00
)
2023-08-30 10:15:35 +02:00
func ( s * composeService ) Publish ( ctx context . Context , project * types . Project , repository string , options api . PublishOptions ) error {
2023-09-15 19:48:21 +02:00
return progress . RunWithTitle ( ctx , func ( ctx context . Context ) error {
return s . publish ( ctx , project , repository , options )
} , s . stdinfo ( ) , "Publishing" )
}
func ( s * composeService ) publish ( ctx context . Context , project * types . Project , repository string , options api . PublishOptions ) error {
2025-01-21 17:50:11 +01:00
accept , err := s . preChecks ( project , options )
2025-01-21 09:51:56 +01:00
if err != nil {
return err
}
2025-01-21 17:50:11 +01:00
if ! accept {
return nil
}
2025-01-21 09:51:56 +01:00
err = s . Push ( ctx , project , api . PushOptions { IgnoreFailures : true , ImageMandatory : true } )
2023-07-05 08:51:03 +02:00
if err != nil {
return err
}
2023-09-15 19:48:21 +02:00
named , err := reference . ParseDockerRef ( repository )
if err != nil {
return err
}
resolver := imagetools . New ( imagetools . Opt {
Auth : s . configFile ( ) ,
} )
2023-12-04 22:39:23 -05:00
var layers [ ] ocipush . Pushable
2025-03-05 11:41:43 +01:00
extFiles := map [ string ] string { }
2023-09-15 19:48:21 +02:00
for _ , file := range project . ComposeFiles {
2025-03-05 11:41:43 +01:00
data , err := processFile ( ctx , file , project , extFiles )
2023-09-15 19:48:21 +02:00
if err != nil {
return err
}
2025-03-05 11:41:43 +01:00
layerDescriptor := ocipush . DescriptorForComposeFile ( file , data )
2023-12-04 22:39:23 -05:00
layers = append ( layers , ocipush . Pushable {
Descriptor : layerDescriptor ,
2025-03-05 11:41:43 +01:00
Data : data ,
2023-12-04 22:39:23 -05:00
} )
2023-10-26 15:34:52 +02:00
}
2023-09-15 19:48:21 +02:00
2025-03-05 11:41:43 +01:00
extLayers , err := processExtends ( ctx , project , extFiles )
if err != nil {
return err
}
layers = append ( layers , extLayers ... )
2025-01-21 09:51:56 +01:00
if options . WithEnvironment {
layers = append ( layers , envFileLayers ( project ) ... )
}
2023-10-26 15:34:52 +02:00
if options . ResolveImageDigests {
yaml , err := s . generateImageDigestsOverride ( ctx , project )
if err != nil {
2023-09-15 19:48:21 +02:00
return err
}
2023-12-05 11:52:35 -05:00
layerDescriptor := ocipush . DescriptorForComposeFile ( "image-digests.yaml" , yaml )
2023-12-04 22:39:23 -05:00
layers = append ( layers , ocipush . Pushable {
Descriptor : layerDescriptor ,
Data : yaml ,
} )
2023-09-15 19:48:21 +02:00
}
2023-07-05 08:51:03 +02:00
fix(publish): add OCI 1.0 fallback support for AWS ECR
Currently, we publish Compose artifacts following the OCI 1.1
specification, which is still in the RC state.
As a result, not all registries support it yet. Most notably,
AWS ECR will reject certain OCI 1.1-compliant requests with
`405 Method Not Supported` with cryptic `Invalid JSON` errors.
This adds initial support for Compose to generate either an
OCI 1.0 or OCI 1.1 compatible manifest. Notably, the OCI 1.0
manifest will be missing the `application/vnd.docker.compose.project`
artifact type, as that does not exist in that version of the
spec. (Less importantly, it uses an empty `ImageConfig`
instead of the newer `application/vnd.oci.empty.v1+json` media
type for the config.)
Currently, this is not exposed as an option (via CLI flags or
env vars). By default, OCI 1.1 is used unless the registry
domain is `amazonaws.com`, which indicates an ECR registry, so
Compose will instead use OCI 1.0.
Moving forward, we should decide how much we want to expose/
support different OCI versions and investigate if there's a
more generic way to feature probe the registry to avoid
maintaining a hardcoded list of domains, which is both tedious
and insufficient.
Signed-off-by: Milas Bowman <milas.bowman@docker.com>
2023-12-01 13:33:00 -05:00
w := progress . ContextWriter ( ctx )
2023-09-15 19:48:21 +02:00
w . Event ( progress . Event {
ID : repository ,
Text : "publishing" ,
Status : progress . Working ,
} )
2023-10-05 11:31:56 -07:00
if ! s . dryRun {
2023-12-04 22:39:23 -05:00
err = ocipush . PushManifest ( ctx , resolver , named , layers , options . OCIVersion )
if err != nil {
2024-06-07 15:34:51 +02:00
w . Event ( progress . Event {
ID : repository ,
Text : "publishing" ,
Status : progress . Error ,
} )
2023-12-04 22:39:23 -05:00
return err
fix(publish): add OCI 1.0 fallback support for AWS ECR
Currently, we publish Compose artifacts following the OCI 1.1
specification, which is still in the RC state.
As a result, not all registries support it yet. Most notably,
AWS ECR will reject certain OCI 1.1-compliant requests with
`405 Method Not Supported` with cryptic `Invalid JSON` errors.
This adds initial support for Compose to generate either an
OCI 1.0 or OCI 1.1 compatible manifest. Notably, the OCI 1.0
manifest will be missing the `application/vnd.docker.compose.project`
artifact type, as that does not exist in that version of the
spec. (Less importantly, it uses an empty `ImageConfig`
instead of the newer `application/vnd.oci.empty.v1+json` media
type for the config.)
Currently, this is not exposed as an option (via CLI flags or
env vars). By default, OCI 1.1 is used unless the registry
domain is `amazonaws.com`, which indicates an ECR registry, so
Compose will instead use OCI 1.0.
Moving forward, we should decide how much we want to expose/
support different OCI versions and investigate if there's a
more generic way to feature probe the registry to avoid
maintaining a hardcoded list of domains, which is both tedious
and insufficient.
Signed-off-by: Milas Bowman <milas.bowman@docker.com>
2023-12-01 13:33:00 -05:00
}
2023-09-15 19:48:21 +02:00
}
w . Event ( progress . Event {
ID : repository ,
Text : "published" ,
Status : progress . Done ,
} )
return nil
2023-07-05 08:51:03 +02:00
}
2023-10-26 15:34:52 +02:00
2025-03-05 11:41:43 +01:00
func processExtends ( ctx context . Context , project * types . Project , extFiles map [ string ] string ) ( [ ] ocipush . Pushable , error ) {
var layers [ ] ocipush . Pushable
moreExtFiles := map [ string ] string { }
for xf , hash := range extFiles {
data , err := processFile ( ctx , xf , project , moreExtFiles )
if err != nil {
return nil , err
}
layerDescriptor := ocipush . DescriptorForComposeFile ( hash , data )
layerDescriptor . Annotations [ "com.docker.compose.extends" ] = "true"
layers = append ( layers , ocipush . Pushable {
Descriptor : layerDescriptor ,
Data : data ,
} )
}
for f , hash := range moreExtFiles {
if _ , ok := extFiles [ f ] ; ok {
delete ( moreExtFiles , f )
}
extFiles [ f ] = hash
}
if len ( moreExtFiles ) > 0 {
extLayers , err := processExtends ( ctx , project , moreExtFiles )
if err != nil {
return nil , err
}
layers = append ( layers , extLayers ... )
}
return layers , nil
}
func processFile ( ctx context . Context , file string , project * types . Project , extFiles map [ string ] string ) ( [ ] byte , error ) {
f , err := os . ReadFile ( file )
if err != nil {
return nil , err
}
base , err := loader . LoadWithContext ( ctx , types . ConfigDetails {
WorkingDir : project . WorkingDir ,
Environment : project . Environment ,
ConfigFiles : [ ] types . ConfigFile {
{
Filename : file ,
Content : f ,
} ,
} ,
} , func ( options * loader . Options ) {
options . SkipValidation = true
options . SkipExtends = true
options . SkipConsistencyCheck = true
options . ResolvePaths = true
} )
if err != nil {
return nil , err
}
for name , service := range base . Services {
if service . Extends == nil {
continue
}
xf := service . Extends . File
if xf == "" {
continue
}
if _ , err = os . Stat ( service . Extends . File ) ; os . IsNotExist ( err ) {
// No local file, while we loaded the project successfully: This is actually a remote resource
continue
}
hash := fmt . Sprintf ( "%x.yaml" , sha256 . Sum256 ( [ ] byte ( xf ) ) )
extFiles [ xf ] = hash
2025-03-07 17:52:16 +01:00
f , err = transform . ReplaceExtendsFile ( f , name , hash )
2025-03-05 11:41:43 +01:00
if err != nil {
return nil , err
}
}
return f , nil
}
2023-10-26 15:34:52 +02:00
func ( s * composeService ) generateImageDigestsOverride ( ctx context . Context , project * types . Project ) ( [ ] byte , error ) {
2023-12-29 11:45:45 +01:00
project , err := project . WithProfiles ( [ ] string { "*" } )
if err != nil {
return nil , err
}
2024-02-28 12:39:24 +01:00
project , err = project . WithImagesResolved ( ImageDigestResolver ( ctx , s . configFile ( ) , s . apiClient ( ) ) )
2023-10-26 15:34:52 +02:00
if err != nil {
return nil , err
}
2023-11-08 10:19:24 +01:00
override := types . Project {
Services : types . Services { } ,
}
for name , service := range project . Services {
override . Services [ name ] = types . ServiceConfig {
2023-10-26 15:34:52 +02:00
Image : service . Image ,
2023-11-08 10:19:24 +01:00
}
2023-10-26 15:34:52 +02:00
}
return override . MarshalYAML ( )
}
2025-01-21 09:51:56 +01:00
2025-03-10 17:26:07 +01:00
//nolint:gocyclo
2025-01-21 17:50:11 +01:00
func ( s * composeService ) preChecks ( project * types . Project , options api . PublishOptions ) ( bool , error ) {
2025-03-10 17:26:07 +01:00
if ok , err := s . checkOnlyBuildSection ( project ) ; ! ok || err != nil {
return false , err
}
if ok , err := s . checkForBindMount ( project ) ; ! ok || err != nil {
2025-03-03 14:51:39 +01:00
return false , err
}
2025-03-10 17:26:07 +01:00
if options . AssumeYes {
return true , nil
}
detectedSecrets , err := s . checkForSensitiveData ( project )
if err != nil {
return false , err
}
if len ( detectedSecrets ) > 0 {
fmt . Println ( "you are about to publish sensitive data within your OCI artifact.\n" +
"please double check that you are not leaking sensitive data" )
for _ , val := range detectedSecrets {
_ , _ = fmt . Fprintln ( s . dockerCli . Out ( ) , val . Type )
_ , _ = fmt . Fprintf ( s . dockerCli . Out ( ) , "%q: %s\n" , val . Key , val . Value )
}
if ok , err := acceptPublishSensitiveData ( s . dockerCli ) ; err != nil || ! ok {
return false , err
}
}
2025-01-21 17:50:11 +01:00
envVariables , err := s . checkEnvironmentVariables ( project , options )
if err != nil {
return false , err
}
2025-03-10 17:26:07 +01:00
if len ( envVariables ) > 0 {
2025-01-21 17:50:11 +01:00
fmt . Println ( "you are about to publish environment variables within your OCI artifact.\n" +
"please double check that you are not leaking sensitive data" )
for key , val := range envVariables {
_ , _ = fmt . Fprintln ( s . dockerCli . Out ( ) , "Service/Config " , key )
for k , v := range val {
_ , _ = fmt . Fprintf ( s . dockerCli . Out ( ) , "%s=%v\n" , k , * v )
2025-01-21 09:51:56 +01:00
}
}
2025-03-10 17:26:07 +01:00
if ok , err := acceptPublishEnvVariables ( s . dockerCli ) ; err != nil || ! ok {
return false , err
2025-03-04 15:49:59 +01:00
}
}
2025-01-21 17:50:11 +01:00
return true , nil
}
func ( s * composeService ) checkEnvironmentVariables ( project * types . Project , options api . PublishOptions ) ( map [ string ] types . MappingWithEquals , error ) {
envVarList := map [ string ] types . MappingWithEquals { }
errorList := map [ string ] [ ] string { }
for _ , service := range project . Services {
if len ( service . EnvFiles ) > 0 {
errorList [ service . Name ] = append ( errorList [ service . Name ] , fmt . Sprintf ( "service %q has env_file declared." , service . Name ) )
}
if len ( service . Environment ) > 0 {
errorList [ service . Name ] = append ( errorList [ service . Name ] , fmt . Sprintf ( "service %q has environment variable(s) declared." , service . Name ) )
envVarList [ service . Name ] = service . Environment
}
}
2025-01-21 09:51:56 +01:00
2025-01-21 17:50:11 +01:00
for _ , config := range project . Configs {
if config . Environment != "" {
errorList [ config . Name ] = append ( errorList [ config . Name ] , fmt . Sprintf ( "config %q is declare as an environment variable." , config . Name ) )
envVarList [ config . Name ] = types . NewMappingWithEquals ( [ ] string { fmt . Sprintf ( "%s=%s" , config . Name , config . Environment ) } )
}
}
if ! options . WithEnvironment && len ( errorList ) > 0 {
errorMsgSuffix := "To avoid leaking sensitive data, you must either explicitly allow the sending of environment variables by using the --with-env flag,\n" +
"or remove sensitive data from your Compose configuration"
errorMsg := ""
for _ , errors := range errorList {
for _ , err := range errors {
errorMsg += fmt . Sprintf ( "%s\n" , err )
2025-01-21 09:51:56 +01:00
}
}
2025-01-21 17:50:11 +01:00
return nil , fmt . Errorf ( "%s%s" , errorMsg , errorMsgSuffix )
2025-01-21 09:51:56 +01:00
}
2025-01-21 17:50:11 +01:00
return envVarList , nil
}
2025-01-21 09:51:56 +01:00
2025-01-21 17:50:11 +01:00
func acceptPublishEnvVariables ( cli command . Cli ) ( bool , error ) {
msg := "Are you ok to publish these environment variables? [y/N]: "
confirm , err := prompt . NewPrompt ( cli . In ( ) , cli . Out ( ) ) . Confirm ( msg , false )
return confirm , err
2025-01-21 09:51:56 +01:00
}
2025-03-10 17:26:07 +01:00
func acceptPublishSensitiveData ( cli command . Cli ) ( bool , error ) {
msg := "Are you ok to publish these sensitive data? [y/N]: "
confirm , err := prompt . NewPrompt ( cli . In ( ) , cli . Out ( ) ) . Confirm ( msg , false )
return confirm , err
}
2025-01-21 09:51:56 +01:00
func envFileLayers ( project * types . Project ) [ ] ocipush . Pushable {
var layers [ ] ocipush . Pushable
for _ , service := range project . Services {
for _ , envFile := range service . EnvFiles {
f , err := os . ReadFile ( envFile . Path )
if err != nil {
// if we can't read the file, skip to the next one
continue
}
layerDescriptor := ocipush . DescriptorForEnvFile ( envFile . Path , f )
layers = append ( layers , ocipush . Pushable {
Descriptor : layerDescriptor ,
Data : f ,
} )
}
}
return layers
}
2025-03-03 14:51:39 +01:00
func ( s * composeService ) checkOnlyBuildSection ( project * types . Project ) ( bool , error ) {
errorList := [ ] string { }
for _ , service := range project . Services {
if service . Image == "" && service . Build != nil {
errorList = append ( errorList , service . Name )
}
}
if len ( errorList ) > 0 {
errMsg := "your Compose stack cannot be published as it only contains a build section for service(s):\n"
for _ , serviceInError := range errorList {
errMsg += fmt . Sprintf ( "- %q\n" , serviceInError )
}
return false , errors . New ( errMsg )
}
return true , nil
}
2025-03-10 17:26:07 +01:00
func ( s * composeService ) checkForBindMount ( project * types . Project ) ( bool , error ) {
for name , config := range project . Services {
for _ , volume := range config . Volumes {
if volume . Type == types . VolumeTypeBind {
return false , fmt . Errorf ( "cannot publish compose file: service %q relies on bind-mount. You should use volumes" , name )
}
}
}
return true , nil
}
func ( s * composeService ) checkForSensitiveData ( project * types . Project ) ( [ ] secrets . DetectedSecret , error ) {
var allFindings [ ] secrets . DetectedSecret
scan := scanner . NewDefaultScanner ( )
// Check all compose files
for _ , file := range project . ComposeFiles {
in , err := composeFileAsByteReader ( file , project )
if err != nil {
return nil , err
}
findings , err := scan . ScanReader ( in )
if err != nil {
return nil , fmt . Errorf ( "failed to scan compose file %s: %w" , file , err )
}
allFindings = append ( allFindings , findings ... )
}
for _ , service := range project . Services {
// Check env files
for _ , envFile := range service . EnvFiles {
findings , err := scan . ScanFile ( envFile . Path )
if err != nil {
return nil , fmt . Errorf ( "failed to scan env file %s: %w" , envFile . Path , err )
}
allFindings = append ( allFindings , findings ... )
}
}
// Check configs defined by files
for _ , config := range project . Configs {
if config . File != "" {
findings , err := scan . ScanFile ( config . File )
if err != nil {
return nil , fmt . Errorf ( "failed to scan config file %s: %w" , config . File , err )
}
allFindings = append ( allFindings , findings ... )
}
}
// Check secrets defined by files
for _ , secret := range project . Secrets {
if secret . File != "" {
findings , err := scan . ScanFile ( secret . File )
if err != nil {
return nil , fmt . Errorf ( "failed to scan secret file %s: %w" , secret . File , err )
}
allFindings = append ( allFindings , findings ... )
}
}
return allFindings , nil
}
func composeFileAsByteReader ( filePath string , project * types . Project ) ( io . Reader , error ) {
composeFile , err := os . ReadFile ( filePath )
if err != nil {
return nil , fmt . Errorf ( "failed to open compose file %s: %w" , filePath , err )
}
base , err := loader . LoadWithContext ( context . TODO ( ) , types . ConfigDetails {
WorkingDir : project . WorkingDir ,
Environment : project . Environment ,
ConfigFiles : [ ] types . ConfigFile {
{
Filename : filePath ,
Content : composeFile ,
} ,
} ,
} , func ( options * loader . Options ) {
options . SkipValidation = true
options . SkipExtends = true
options . SkipConsistencyCheck = true
options . ResolvePaths = true
options . SkipInterpolation = true
options . SkipResolveEnvironment = true
} )
if err != nil {
return nil , err
}
in , err := base . MarshalYAML ( )
if err != nil {
return nil , err
}
return bytes . NewBuffer ( in ) , nil
}