diff --git a/go.mod b/go.mod index 16a40d3..d6691b8 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/apex/log v1.9.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect @@ -67,9 +68,9 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect github.com/docker/cli v28.2.2+incompatible // indirect - github.com/docker/docker v28.2.2+incompatible // indirect + github.com/docker/docker v28.5.1+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect - github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -78,7 +79,6 @@ require ( github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-test/deep v1.1.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.18.0 // indirect @@ -90,6 +90,7 @@ require ( github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect diff --git a/go.sum b/go.sum index 6fd5278..a369c83 100644 --- a/go.sum +++ b/go.sum @@ -21,9 +21,8 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -52,12 +51,12 @@ github.com/docker/cli v28.2.2+incompatible h1:qzx5BNUDFqlvyq4AHzdNB7gSyVTmU4cgsy github.com/docker/cli v28.2.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= -github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -84,8 +83,6 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -119,8 +116,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= @@ -158,8 +153,8 @@ github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -239,8 +234,6 @@ github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zd github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIjVWss0= github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= @@ -283,26 +276,19 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 h1:6/3JGEh1C88g7m+qzzTbl3A0FtsLguXieqofVLU/JAo= golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= @@ -310,7 +296,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -330,8 +315,6 @@ golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= diff --git a/lib/builds/builder_agent/dockerfile_rewrite_test.go b/lib/builds/builder_agent/dockerfile_rewrite_test.go new file mode 100644 index 0000000..88e2c00 --- /dev/null +++ b/lib/builds/builder_agent/dockerfile_rewrite_test.go @@ -0,0 +1,265 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockRegistryServer creates a test server that responds to manifest HEAD requests. +// The availableImages map determines which images return 200 (found) vs 404 (not found). +func mockRegistryServer(availableImages map[string]bool) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Parse the path to extract repo and tag + // Expected format: /v2/{repo}/manifests/{tag} + path := r.URL.Path + if !strings.HasPrefix(path, "/v2/") || !strings.Contains(path, "/manifests/") { + http.NotFound(w, r) + return + } + + // Extract repo and tag + parts := strings.SplitN(strings.TrimPrefix(path, "/v2/"), "/manifests/", 2) + if len(parts) != 2 { + http.NotFound(w, r) + return + } + + imageRef := parts[0] + ":" + parts[1] + + if availableImages[imageRef] { + w.WriteHeader(http.StatusOK) + } else { + http.NotFound(w, r) + } + })) +} + +func TestRewriteDockerfileFROMs(t *testing.T) { + // Create a mock registry with specific images available + availableImages := map[string]bool{ + "onkernel/nodejs22-base:0.1.1": true, + "onkernel/python311-base:0.1.1": true, + } + server := mockRegistryServer(availableImages) + defer server.Close() + + // Extract host from server URL (remove http://) + registryURL := strings.TrimPrefix(server.URL, "http://") + + tests := []struct { + name string + dockerfile string + expectedCount int + expected string + }{ + { + name: "simple FROM rewrite when image exists locally", + dockerfile: `FROM onkernel/nodejs22-base:0.1.1 +RUN echo hello`, + expectedCount: 1, + expected: `FROM ` + registryURL + `/onkernel/nodejs22-base:0.1.1 +RUN echo hello`, + }, + { + name: "FROM with docker.io prefix", + dockerfile: `FROM docker.io/onkernel/nodejs22-base:0.1.1 +RUN echo hello`, + expectedCount: 1, + expected: `FROM ` + registryURL + `/onkernel/nodejs22-base:0.1.1 +RUN echo hello`, + }, + { + name: "FROM with AS alias", + dockerfile: `FROM onkernel/nodejs22-base:0.1.1 AS builder +RUN npm install +FROM onkernel/nodejs22-base:0.1.1 AS runtime +COPY --from=builder /app /app`, + expectedCount: 2, + expected: `FROM ` + registryURL + `/onkernel/nodejs22-base:0.1.1 AS builder +RUN npm install +FROM ` + registryURL + `/onkernel/nodejs22-base:0.1.1 AS runtime +COPY --from=builder /app /app`, + }, + { + name: "FROM with --platform flag", + dockerfile: `FROM --platform=linux/amd64 onkernel/nodejs22-base:0.1.1 +RUN echo hello`, + expectedCount: 1, + expected: `FROM --platform=linux/amd64 ` + registryURL + `/onkernel/nodejs22-base:0.1.1 +RUN echo hello`, + }, + { + name: "no rewrite when image not in local registry", + dockerfile: `FROM alpine:3.21 +RUN echo hello`, + expectedCount: 0, + expected: `FROM alpine:3.21 +RUN echo hello`, + }, + { + name: "preserves comments and whitespace", + dockerfile: `# This is a comment +FROM onkernel/nodejs22-base:0.1.1 + +# Another comment +RUN echo hello`, + expectedCount: 1, + expected: `# This is a comment +FROM ` + registryURL + `/onkernel/nodejs22-base:0.1.1 + +# Another comment +RUN echo hello`, + }, + { + name: "lowercase from is rewritten", + dockerfile: `from onkernel/nodejs22-base:0.1.1 +RUN echo hello`, + expectedCount: 1, + expected: `from ` + registryURL + `/onkernel/nodejs22-base:0.1.1 +RUN echo hello`, + }, + { + name: "scratch image is not rewritten", + dockerfile: `FROM scratch +COPY binary /`, + expectedCount: 0, + expected: `FROM scratch +COPY binary /`, + }, + { + name: "already local registry reference is not rewritten", + dockerfile: `FROM ` + registryURL + `/onkernel/nodejs22-base:0.1.1 +RUN echo hello`, + expectedCount: 0, + expected: `FROM ` + registryURL + `/onkernel/nodejs22-base:0.1.1 +RUN echo hello`, + }, + { + name: "inter-stage FROM reference is not rewritten", + dockerfile: `FROM onkernel/nodejs22-base:0.1.1 AS builder +RUN npm install +FROM builder +COPY --from=builder /app /app`, + expectedCount: 1, + expected: `FROM ` + registryURL + `/onkernel/nodejs22-base:0.1.1 AS builder +RUN npm install +FROM builder +COPY --from=builder /app /app`, + }, + { + name: "inter-stage reference case insensitive", + dockerfile: `FROM onkernel/nodejs22-base:0.1.1 AS Builder +RUN npm install +FROM builder +COPY --from=builder /app /app`, + expectedCount: 1, + expected: `FROM ` + registryURL + `/onkernel/nodejs22-base:0.1.1 AS Builder +RUN npm install +FROM builder +COPY --from=builder /app /app`, + }, + { + name: "variable reference FROM is not rewritten", + dockerfile: `ARG BASE_IMAGE=onkernel/nodejs22-base:0.1.1 +FROM ${BASE_IMAGE} +RUN echo hello`, + expectedCount: 0, + expected: `ARG BASE_IMAGE=onkernel/nodejs22-base:0.1.1 +FROM ${BASE_IMAGE} +RUN echo hello`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create temp file + tmpDir := t.TempDir() + dockerfilePath := filepath.Join(tmpDir, "Dockerfile") + err := os.WriteFile(dockerfilePath, []byte(tt.dockerfile), 0644) + require.NoError(t, err) + + // Run rewrite (insecure=true for http test server) + count, err := rewriteDockerfileFROMs(dockerfilePath, registryURL, true, "") + require.NoError(t, err) + assert.Equal(t, tt.expectedCount, count) + + // Check result + result, err := os.ReadFile(dockerfilePath) + require.NoError(t, err) + assert.Equal(t, tt.expected, string(result)) + }) + } +} + +func TestNormalizeImageRef(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"docker.io/onkernel/nodejs22-base:0.1.1", "onkernel/nodejs22-base:0.1.1"}, + {"docker.io/library/alpine:3.21", "alpine:3.21"}, + {"onkernel/nodejs22-base:0.1.1", "onkernel/nodejs22-base:0.1.1"}, + {"alpine:3.21", "alpine:3.21"}, + {"library/alpine:3.21", "alpine:3.21"}, + {"nginx", "nginx"}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := normalizeImageRef(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestCheckImageExistsInRegistry(t *testing.T) { + availableImages := map[string]bool{ + "myimage:v1": true, + "org/myimage:latest": true, + } + server := mockRegistryServer(availableImages) + defer server.Close() + + registryURL := strings.TrimPrefix(server.URL, "http://") + + tests := []struct { + name string + imageRef string + expected bool + }{ + { + name: "image exists with tag", + imageRef: "myimage:v1", + expected: true, + }, + { + name: "image exists with org and latest tag", + imageRef: "org/myimage:latest", + expected: true, + }, + { + name: "image does not exist", + imageRef: "notfound:v1", + expected: false, + }, + { + name: "image without tag defaults to latest", + imageRef: "org/myimage", + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := checkImageExistsInRegistry(registryURL, tt.imageRef, true, "") + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/lib/builds/builder_agent/main.go b/lib/builds/builder_agent/main.go index beb5b18..ced5489 100644 --- a/lib/builds/builder_agent/main.go +++ b/lib/builds/builder_agent/main.go @@ -13,6 +13,7 @@ import ( "bytes" "context" "crypto/sha256" + "crypto/tls" "encoding/base64" "encoding/hex" "encoding/json" @@ -20,6 +21,7 @@ import ( "io" "log" "net" + "net/http" "os" "os/exec" "path/filepath" @@ -50,8 +52,8 @@ type BuildConfig struct { Secrets []SecretRef `json:"secrets,omitempty"` TimeoutSeconds int `json:"timeout_seconds"` NetworkMode string `json:"network_mode"` - IsAdminBuild bool `json:"is_admin_build,omitempty"` - GlobalCacheKey string `json:"global_cache_key,omitempty"` + IsAdminBuild bool `json:"is_admin_build,omitempty"` + GlobalCacheKey string `json:"global_cache_key,omitempty"` } // SecretRef references a secret to inject during build @@ -438,6 +440,23 @@ func runBuildProcess() { log.Println("Using Dockerfile from source") } + // Rewrite Dockerfile FROM instructions to use locally mirrored base images + // This avoids pulling base image layers from Docker Hub during builds + // The function auto-detects which images exist in the local registry + registryHost := config.RegistryURL + if strings.HasPrefix(registryHost, "https://") { + registryHost = strings.TrimPrefix(registryHost, "https://") + } else if strings.HasPrefix(registryHost, "http://") { + registryHost = strings.TrimPrefix(registryHost, "http://") + } + + rewriteCount, err := rewriteDockerfileFROMs(dockerfilePath, registryHost, config.RegistryInsecure, config.RegistryToken) + if err != nil { + log.Printf("Warning: failed to rewrite Dockerfile FROMs: %v", err) + } else if rewriteCount > 0 { + log.Printf("Rewrote %d FROM instruction(s) to use local base images", rewriteCount) + } + // Compute provenance provenance := computeProvenance(config) @@ -728,11 +747,14 @@ func runBuild(ctx context.Context, config *BuildConfig, logWriter io.Writer) (st } // Export cache based on build type + // Note: image-manifest=true ensures layer blobs are stored in the registry cache image + // rather than as references to external registries (e.g., docker.io). This is critical + // for cache hits in ephemeral BuildKit instances that don't have local layer storage. if config.IsAdminBuild { // Admin build: export to global cache if config.GlobalCacheKey != "" { globalCacheRef := fmt.Sprintf("%s/cache/global/%s", registryHost, config.GlobalCacheKey) - cacheOpts := "type=registry,ref=" + globalCacheRef + ",mode=max" + cacheOpts := "type=registry,ref=" + globalCacheRef + ",mode=max,image-manifest=true,oci-mediatypes=true" if useInsecureFlag { cacheOpts += ",registry.insecure=true" } @@ -743,7 +765,7 @@ func runBuild(ctx context.Context, config *BuildConfig, logWriter io.Writer) (st // Regular build: export to tenant cache if config.CacheScope != "" { tenantCacheRef := fmt.Sprintf("%s/cache/%s", registryHost, config.CacheScope) - cacheOpts := "type=registry,ref=" + tenantCacheRef + ",mode=max" + cacheOpts := "type=registry,ref=" + tenantCacheRef + ",mode=max,image-manifest=true,oci-mediatypes=true" if useInsecureFlag { cacheOpts += ",registry.insecure=true" } @@ -891,3 +913,210 @@ func getBuildkitVersion() string { out, _ := cmd.Output() return strings.TrimSpace(string(out)) } + +// rewriteDockerfileFROMs rewrites FROM instructions in a Dockerfile to use locally +// mirrored base images instead of pulling from external registries like Docker Hub. +// This is the key mechanism for avoiding Docker Hub downloads during builds. +// +// For each FROM instruction, if the base image exists in the local registry, it's +// rewritten to use the local registry reference. For example: +// +// FROM onkernel/nodejs22-base:0.1.1 +// +// becomes: +// +// FROM 172.30.0.1:8080/onkernel/nodejs22-base:0.1.1 +// +// The function handles multi-stage builds (multiple FROM instructions) and preserves +// AS aliases and other Dockerfile syntax. +func rewriteDockerfileFROMs(dockerfilePath, registryURL string, insecure bool, registryToken string) (int, error) { + content, err := os.ReadFile(dockerfilePath) + if err != nil { + return 0, fmt.Errorf("read dockerfile: %w", err) + } + + lines := strings.Split(string(content), "\n") + rewriteCount := 0 + stageNames := make(map[string]bool) + + for i, line := range lines { + trimmed := strings.TrimSpace(line) + + // Skip empty lines and comments + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + // Check for FROM instruction (case insensitive) + upper := strings.ToUpper(trimmed) + if !strings.HasPrefix(upper, "FROM ") { + continue + } + + // Parse the FROM instruction + // Formats: FROM image, FROM image AS name, FROM image:tag, FROM image:tag AS name + // Also: FROM --platform=xxx image ... + parts := strings.Fields(trimmed) + if len(parts) < 2 { + continue + } + + // Find the image reference (skip FROM and any flags like --platform) + imageIdx := 1 + for imageIdx < len(parts) && strings.HasPrefix(parts[imageIdx], "--") { + imageIdx++ + } + if imageIdx >= len(parts) { + continue + } + + imageRef := parts[imageIdx] + + // Record AS alias if present + for j := imageIdx + 1; j < len(parts)-1; j++ { + if strings.EqualFold(parts[j], "AS") { + stageNames[strings.ToLower(parts[j+1])] = true + break + } + } + + // Skip if already referencing the local registry + if strings.HasPrefix(imageRef, registryURL+"/") { + continue + } + + // Skip scratch image (special case in Docker) + if imageRef == "scratch" { + continue + } + + // Skip inter-stage references (e.g. FROM builder) + if stageNames[strings.ToLower(imageRef)] { + continue + } + + // Skip variable references that can't be resolved + if strings.Contains(imageRef, "${") { + continue + } + + // Normalize the image reference + // Docker Hub images can be referenced as: + // - "nginx" (library image) + // - "nginx:1.21" + // - "library/nginx:1.21" + // - "docker.io/library/nginx:1.21" + // - "onkernel/nodejs22-base:0.1.1" + // - "docker.io/onkernel/nodejs22-base:0.1.1" + normalizedRef := normalizeImageRef(imageRef) + + // Check if the image exists in the local registry + if !checkImageExistsInRegistry(registryURL, normalizedRef, insecure, registryToken) { + log.Printf("Base image not found locally, will pull from upstream: %s", imageRef) + continue + } + + // Build the new image reference with the local registry + newImageRef := fmt.Sprintf("%s/%s", registryURL, normalizedRef) + + // Reconstruct the FROM line with the new image reference + parts[imageIdx] = newImageRef + newLine := strings.Join(parts, " ") + + // Preserve original indentation + indent := line[:len(line)-len(strings.TrimLeft(line, " \t"))] + lines[i] = indent + newLine + + log.Printf("Rewriting FROM: %s -> %s", imageRef, newImageRef) + rewriteCount++ + } + + if rewriteCount == 0 { + return 0, nil + } + + // Write the modified Dockerfile back + newContent := strings.Join(lines, "\n") + if err := os.WriteFile(dockerfilePath, []byte(newContent), 0644); err != nil { + return 0, fmt.Errorf("write dockerfile: %w", err) + } + + return rewriteCount, nil +} + +// checkImageExistsInRegistry checks if an image exists in the local registry +// by making a HEAD request to the manifest endpoint. +func checkImageExistsInRegistry(registryURL, imageRef string, insecure bool, registryToken string) bool { + // Parse the image reference to extract repo and tag + repo := imageRef + tag := "latest" + + // Handle digest references (repo@sha256:...) + if strings.Contains(imageRef, "@") { + parts := strings.SplitN(imageRef, "@", 2) + repo = parts[0] + tag = parts[1] // This will be the full digest like sha256:abc123 + } else if strings.Contains(imageRef, ":") { + // Handle tag references (repo:tag) + lastColon := strings.LastIndex(imageRef, ":") + repo = imageRef[:lastColon] + tag = imageRef[lastColon+1:] + } + + // Build the manifest URL + scheme := "https" + if insecure { + scheme = "http" + } + url := fmt.Sprintf("%s://%s/v2/%s/manifests/%s", scheme, registryURL, repo, tag) + + // Create HTTP client with appropriate TLS settings + client := &http.Client{ + Timeout: 5 * time.Second, + } + if insecure { + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + + // Make HEAD request to check if manifest exists + req, err := http.NewRequest(http.MethodHead, url, nil) + if err != nil { + log.Printf("Failed to create request for %s: %v", url, err) + return false + } + + // Accept OCI and Docker manifest types + req.Header.Set("Accept", "application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json") + + if registryToken != "" { + req.Header.Set("Authorization", "Bearer "+registryToken) + } + + resp, err := client.Do(req) + if err != nil { + log.Printf("Failed to check image %s in registry: %v", imageRef, err) + return false + } + defer resp.Body.Close() + + // 200 means the image exists + return resp.StatusCode == http.StatusOK +} + +// normalizeImageRef normalizes a Docker image reference for consistent lookup. +// It handles various forms of Docker Hub image references: +// - "nginx" -> "nginx" (library images stay as-is for simple lookup) +// - "nginx:1.21" -> "nginx:1.21" +// - "docker.io/library/nginx:1.21" -> "nginx:1.21" +// - "docker.io/onkernel/nodejs22-base:0.1.1" -> "onkernel/nodejs22-base:0.1.1" +func normalizeImageRef(ref string) string { + // Strip docker.io/ prefix if present + ref = strings.TrimPrefix(ref, "docker.io/") + + // Strip library/ prefix for official images + ref = strings.TrimPrefix(ref, "library/") + + return ref +} diff --git a/lib/builds/cache.go b/lib/builds/cache.go index ff3e26a..f47e331 100644 --- a/lib/builds/cache.go +++ b/lib/builds/cache.go @@ -100,8 +100,10 @@ func (k *CacheKey) ImportCacheArg() string { } // ExportCacheArg returns the BuildKit --export-cache argument +// Uses image-manifest=true to ensure layer blobs are stored in the cache image +// rather than as external references, enabling cache hits in ephemeral BuildKit instances. func (k *CacheKey) ExportCacheArg() string { - return fmt.Sprintf("type=registry,ref=%s,mode=max", k.Reference) + return fmt.Sprintf("type=registry,ref=%s,mode=max,image-manifest=true,oci-mediatypes=true", k.Reference) } // normalizeCacheScope normalizes a cache scope to only contain safe characters diff --git a/lib/builds/cache_test.go b/lib/builds/cache_test.go index d51fb7c..7f3637b 100644 --- a/lib/builds/cache_test.go +++ b/lib/builds/cache_test.go @@ -103,7 +103,7 @@ func TestCacheKey_Args(t *testing.T) { assert.Equal(t, "type=registry,ref=localhost:8080/cache/tenant/nodejs/abc123", importArg) exportArg := key.ExportCacheArg() - assert.Equal(t, "type=registry,ref=localhost:8080/cache/tenant/nodejs/abc123,mode=max", exportArg) + assert.Equal(t, "type=registry,ref=localhost:8080/cache/tenant/nodejs/abc123,mode=max,image-manifest=true,oci-mediatypes=true", exportArg) } func TestValidateCacheScope(t *testing.T) { diff --git a/lib/builds/dockerfile.go b/lib/builds/dockerfile.go new file mode 100644 index 0000000..a968a57 --- /dev/null +++ b/lib/builds/dockerfile.go @@ -0,0 +1,138 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// ParseDockerfileFROMs extracts and deduplicates base image references from +// Dockerfile content. It reuses the same parsing logic as the builder agent's +// rewriteDockerfileFROMs: split lines, find FROM, skip flags/comments/scratch, +// normalize refs. Inter-stage references (FROM builder) and variable references +// (${VAR}) are skipped since they can't be resolved at parse time. +func ParseDockerfileFROMs(content string) []string { + lines := strings.Split(content, "\n") + + // Track stage names so we can skip inter-stage FROM references + stageNames := make(map[string]bool) + seen := make(map[string]bool) + var refs []string + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + // Skip empty lines and comments + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + // Check for FROM instruction (case insensitive) + upper := strings.ToUpper(trimmed) + if !strings.HasPrefix(upper, "FROM ") { + continue + } + + parts := strings.Fields(trimmed) + if len(parts) < 2 { + continue + } + + // Find the image reference (skip FROM and any flags like --platform) + imageIdx := 1 + for imageIdx < len(parts) && strings.HasPrefix(parts[imageIdx], "--") { + imageIdx++ + } + if imageIdx >= len(parts) { + continue + } + + imageRef := parts[imageIdx] + + // Record AS alias if present + for j := imageIdx + 1; j < len(parts)-1; j++ { + if strings.EqualFold(parts[j], "AS") { + stageNames[strings.ToLower(parts[j+1])] = true + break + } + } + + // Skip scratch + if imageRef == "scratch" { + continue + } + + // Skip inter-stage references (e.g. FROM builder) + if stageNames[strings.ToLower(imageRef)] { + continue + } + + // Skip variable references that can't be resolved + if strings.Contains(imageRef, "${") { + continue + } + + // Normalize the image reference (same logic as builder agent) + normalized := normalizeImageRef(imageRef) + + if !seen[normalized] { + seen[normalized] = true + refs = append(refs, normalized) + } + } + + return refs +} + +// normalizeImageRef normalizes a Docker image reference by stripping +// docker.io/ and library/ prefixes. This matches the builder agent's +// normalizeImageRef function. +func normalizeImageRef(ref string) string { + ref = strings.TrimPrefix(ref, "docker.io/") + ref = strings.TrimPrefix(ref, "library/") + return ref +} + +// ExtractDockerfileFromTarball reads just the Dockerfile entry from a .tar.gz +// archive and returns its content as a string. It looks for entries named +// "Dockerfile" or "./Dockerfile" at the root of the archive. +func ExtractDockerfileFromTarball(tarballPath string) (string, error) { + f, err := os.Open(tarballPath) + if err != nil { + return "", fmt.Errorf("open tarball: %w", err) + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + return "", fmt.Errorf("create gzip reader: %w", err) + } + defer gz.Close() + + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return "", fmt.Errorf("read tar entry: %w", err) + } + + // Match Dockerfile at root (with or without ./ prefix) + name := filepath.Clean(hdr.Name) + if name == "Dockerfile" { + data, err := io.ReadAll(tr) + if err != nil { + return "", fmt.Errorf("read Dockerfile from tarball: %w", err) + } + return string(data), nil + } + } + + return "", fmt.Errorf("Dockerfile not found in tarball") +} diff --git a/lib/builds/dockerfile_test.go b/lib/builds/dockerfile_test.go new file mode 100644 index 0000000..620829f --- /dev/null +++ b/lib/builds/dockerfile_test.go @@ -0,0 +1,230 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "os" + "path/filepath" + "testing" +) + +func TestParseDockerfileFROMs_SingleFROM(t *testing.T) { + content := `FROM onkernel/nodejs22-base:0.1.1 +RUN echo hello +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "onkernel/nodejs22-base:0.1.1" { + t.Errorf("expected onkernel/nodejs22-base:0.1.1, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_MultiStage(t *testing.T) { + content := `FROM golang:1.21 AS builder +RUN go build -o /app . + +FROM alpine:3.21 +COPY --from=builder /app /app +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 2 { + t.Fatalf("expected 2 refs, got %d: %v", len(refs), refs) + } + if refs[0] != "golang:1.21" { + t.Errorf("expected golang:1.21, got %s", refs[0]) + } + if refs[1] != "alpine:3.21" { + t.Errorf("expected alpine:3.21, got %s", refs[1]) + } +} + +func TestParseDockerfileFROMs_DockerIONormalization(t *testing.T) { + content := `FROM docker.io/library/alpine:3.21 +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "alpine:3.21" { + t.Errorf("expected alpine:3.21, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_PlatformFlag(t *testing.T) { + content := `FROM --platform=linux/amd64 node:20-alpine +RUN npm install +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "node:20-alpine" { + t.Errorf("expected node:20-alpine, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_SkipScratch(t *testing.T) { + content := `FROM golang:1.21 AS builder +RUN go build -o /app . + +FROM scratch +COPY --from=builder /app /app +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "golang:1.21" { + t.Errorf("expected golang:1.21, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_SkipStageReferences(t *testing.T) { + content := `FROM node:20 AS deps +RUN npm ci + +FROM node:20 AS builder +COPY --from=deps /app/node_modules ./node_modules +RUN npm run build + +FROM builder +CMD ["node", "dist/index.js"] +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref (deduplicated), got %d: %v", len(refs), refs) + } + if refs[0] != "node:20" { + t.Errorf("expected node:20, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_SkipVariableReferences(t *testing.T) { + content := `ARG BASE_IMAGE=node:20 +FROM ${BASE_IMAGE} +RUN echo hello +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 0 { + t.Fatalf("expected 0 refs (variable), got %d: %v", len(refs), refs) + } +} + +func TestParseDockerfileFROMs_Deduplication(t *testing.T) { + content := `FROM alpine:3.21 AS stage1 +RUN echo one + +FROM alpine:3.21 AS stage2 +RUN echo two +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref (deduplicated), got %d: %v", len(refs), refs) + } + if refs[0] != "alpine:3.21" { + t.Errorf("expected alpine:3.21, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_CommentsAndEmptyLines(t *testing.T) { + content := `# Build stage +FROM golang:1.21 + +# This is a comment +# FROM fake:image + +RUN echo hello +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "golang:1.21" { + t.Errorf("expected golang:1.21, got %s", refs[0]) + } +} + +func TestExtractDockerfileFromTarball(t *testing.T) { + // Create a temp tarball with a Dockerfile + dir := t.TempDir() + tarballPath := filepath.Join(dir, "source.tar.gz") + + dockerfileContent := "FROM alpine:3.21\nRUN echo hello\n" + createTarball(t, tarballPath, map[string]string{ + "Dockerfile": dockerfileContent, + "main.go": "package main\n", + }) + + content, err := ExtractDockerfileFromTarball(tarballPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if content != dockerfileContent { + t.Errorf("expected %q, got %q", dockerfileContent, content) + } +} + +func TestExtractDockerfileFromTarball_NotFound(t *testing.T) { + dir := t.TempDir() + tarballPath := filepath.Join(dir, "source.tar.gz") + + createTarball(t, tarballPath, map[string]string{ + "main.go": "package main\n", + }) + + _, err := ExtractDockerfileFromTarball(tarballPath) + if err == nil { + t.Fatal("expected error for missing Dockerfile") + } +} + +func TestExtractDockerfileFromTarball_DotSlashPrefix(t *testing.T) { + dir := t.TempDir() + tarballPath := filepath.Join(dir, "source.tar.gz") + + dockerfileContent := "FROM node:20\nRUN npm install\n" + createTarball(t, tarballPath, map[string]string{ + "./Dockerfile": dockerfileContent, + }) + + content, err := ExtractDockerfileFromTarball(tarballPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if content != dockerfileContent { + t.Errorf("expected %q, got %q", dockerfileContent, content) + } +} + +// createTarball creates a .tar.gz file with the given files (name -> content). +func createTarball(t *testing.T, path string, files map[string]string) { + t.Helper() + + f, err := os.Create(path) + if err != nil { + t.Fatalf("create tarball file: %v", err) + } + defer f.Close() + + gw := gzip.NewWriter(f) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0644, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("write tar header for %s: %v", name, err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatalf("write tar content for %s: %v", name, err) + } + } +} diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 3a612ba..9321ca1 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -248,6 +248,30 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc } } + // Add pull access for base image repos so the builder agent can + // detect mirrored images via checkImageExistsInRegistry + dockerfileContent := req.Dockerfile + if dockerfileContent == "" { + tarballPath := m.paths.BuildSourceDir(id) + "/source.tar.gz" + if content, err := ExtractDockerfileFromTarball(tarballPath); err == nil { + dockerfileContent = content + } + } + if dockerfileContent != "" { + refs := ParseDockerfileFROMs(dockerfileContent) + seen := make(map[string]bool) + for _, ref := range refs { + repo := ref + if idx := strings.LastIndex(repo, ":"); idx > 0 { + repo = repo[:idx] + } + if !seen[repo] { + seen[repo] = true + repoAccess = append(repoAccess, RepoPermission{Repo: repo, Scope: "pull"}) + } + } + } + registryToken, err := m.tokenGenerator.GenerateToken(id, repoAccess, tokenTTL) if err != nil { deleteBuild(m.paths, id) @@ -315,6 +339,13 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques buildCtx, cancel := context.WithTimeout(ctx, time.Duration(policy.TimeoutSeconds)*time.Second) defer cancel() + // Mirror base images for admin builds before launching the VM + if req.IsAdminBuild { + if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { + m.logger.Warn("failed to mirror base images", "id", id, "error", err) + } + } + // Run the build in a builder VM result, err := m.executeBuild(buildCtx, id, req, policy) @@ -1129,6 +1160,30 @@ func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) err } } + // Add pull access for base image repos so the builder agent can + // detect mirrored images via checkImageExistsInRegistry + dockerfileContent := req.Dockerfile + if dockerfileContent == "" { + tarballPath := m.paths.BuildSourceDir(buildID) + "/source.tar.gz" + if content, err := ExtractDockerfileFromTarball(tarballPath); err == nil { + dockerfileContent = content + } + } + if dockerfileContent != "" { + refs := ParseDockerfileFROMs(dockerfileContent) + seen := make(map[string]bool) + for _, ref := range refs { + repo := ref + if idx := strings.LastIndex(repo, ":"); idx > 0 { + repo = repo[:idx] + } + if !seen[repo] { + seen[repo] = true + repoAccess = append(repoAccess, RepoPermission{Repo: repo, Scope: "pull"}) + } + } + } + // Generate fresh registry token registryToken, err := m.tokenGenerator.GenerateToken(buildID, repoAccess, tokenTTL) if err != nil { diff --git a/lib/builds/mirror.go b/lib/builds/mirror.go new file mode 100644 index 0000000..dc39f4b --- /dev/null +++ b/lib/builds/mirror.go @@ -0,0 +1,86 @@ +package builds + +import ( + "context" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/kernel/hypeman/lib/images" +) + +// mirrorBaseImagesForBuild extracts base image references from the build's +// Dockerfile and mirrors each one to the local registry. This allows the +// builder agent's rewriteDockerfileFROMs to find the images locally and +// rewrite FROM lines to pull from the local registry instead of Docker Hub. +// +// Individual mirror failures are logged but do not fail the build (graceful +// degradation — the builder will simply pull from upstream as before). +func (m *manager) mirrorBaseImagesForBuild(ctx context.Context, id string, req CreateBuildRequest) error { + // Get Dockerfile content: prefer inline Dockerfile, fall back to tarball + var dockerfileContent string + if req.Dockerfile != "" { + dockerfileContent = req.Dockerfile + } else { + tarballPath := m.paths.BuildSourceDir(id) + "/source.tar.gz" + content, err := ExtractDockerfileFromTarball(tarballPath) + if err != nil { + m.logger.Warn("could not extract Dockerfile from tarball for mirroring", + "id", id, "error", err) + return nil + } + dockerfileContent = content + } + + // Parse FROM references + refs := ParseDockerfileFROMs(dockerfileContent) + if len(refs) == 0 { + return nil + } + + m.logger.Info("mirroring base images for admin build", "id", id, "images", refs) + + // Generate a scoped registry token that grants push access to the base + // image repos. The local registry requires JWT auth for all operations; + // go-containerregistry uses this via the Docker token auth flow (Basic + // auth username = JWT → /v2/token validates and returns bearer token). + // Build repo permissions. The Docker token scope uses the repo name without + // the tag (e.g. "onkernel/nodejs22-base", not "onkernel/nodejs22-base:0.1.1"). + seen := make(map[string]bool) + var repoPerms []RepoPermission + for _, ref := range refs { + repo := ref + if idx := strings.LastIndex(repo, ":"); idx > 0 { + repo = repo[:idx] + } + if !seen[repo] { + seen[repo] = true + repoPerms = append(repoPerms, RepoPermission{Repo: repo, Scope: "push"}) + } + } + registryToken, err := m.tokenGenerator.GenerateToken(id, repoPerms, 10*time.Minute) + if err != nil { + m.logger.Warn("failed to generate registry token for mirroring", + "id", id, "error", err) + return nil + } + // go-containerregistry's basicTransport only sends Basic auth when BOTH + // Username and Password are non-empty. The password value doesn't matter — + // our token handler extracts the JWT from the username field only. + authConfig := &authn.AuthConfig{Username: registryToken, Password: "x"} + + for _, ref := range refs { + result, err := images.MirrorBaseImage(ctx, m.config.RegistryURL, images.MirrorRequest{ + SourceImage: ref, + }, authConfig) + if err != nil { + m.logger.Warn("failed to mirror base image", + "id", id, "image", ref, "error", err) + continue + } + m.logger.Info("mirrored base image", + "id", id, "image", ref, "local_ref", result.LocalRef, "digest", result.Digest) + } + + return nil +} diff --git a/lib/images/mirror.go b/lib/images/mirror.go new file mode 100644 index 0000000..6508c61 --- /dev/null +++ b/lib/images/mirror.go @@ -0,0 +1,130 @@ +package images + +import ( + "context" + "fmt" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// MirrorRequest contains the parameters for mirroring a base image +type MirrorRequest struct { + // SourceImage is the full image reference to pull from (e.g., "docker.io/onkernel/nodejs22-base:0.1.1") + SourceImage string +} + +// MirrorResult contains the result of a mirror operation +type MirrorResult struct { + // SourceImage is the original image reference + SourceImage string `json:"source_image"` + // LocalRef is the local registry reference (e.g., "onkernel/nodejs22-base:0.1.1") + LocalRef string `json:"local_ref"` + // Digest is the image digest + Digest string `json:"digest"` +} + +// MirrorBaseImage pulls an image from an external registry and pushes it to the +// local registry with the same normalized name. This enables Dockerfile FROM rewriting +// to use locally mirrored base images instead of pulling from Docker Hub. +// +// For example, mirroring "docker.io/onkernel/nodejs22-base:0.1.1" will create +// "onkernel/nodejs22-base:0.1.1" in the local registry. +func MirrorBaseImage(ctx context.Context, registryURL string, req MirrorRequest, authConfig *authn.AuthConfig) (*MirrorResult, error) { + // Parse source reference + srcRef, err := name.ParseReference(req.SourceImage) + if err != nil { + return nil, fmt.Errorf("parse source image reference: %w", err) + } + + // Pull the image from source + img, err := remote.Image(srcRef, + remote.WithContext(ctx), + remote.WithAuthFromKeychain(authn.DefaultKeychain), + remote.WithPlatform(currentPlatform())) + if err != nil { + return nil, fmt.Errorf("pull source image: %w", wrapRegistryError(err)) + } + + // Get the digest + digest, err := img.Digest() + if err != nil { + return nil, fmt.Errorf("get image digest: %w", err) + } + + // Build the local reference under bases/ namespace + // Normalize the source to strip docker.io/ prefix for cleaner local refs + localRef := normalizeToLocalRef(srcRef) + + // Strip any scheme from registry URL + registryHost := stripScheme(registryURL) + + // Build full destination reference + dstRefStr := fmt.Sprintf("%s/%s", registryHost, localRef) + dstRef, err := name.ParseReference(dstRefStr) + if err != nil { + return nil, fmt.Errorf("parse destination reference: %w", err) + } + + // Push to local registry + // For insecure registries, we need to use the insecure transport + opts := []remote.Option{ + remote.WithContext(ctx), + } + + // If authConfig is provided, use it + if authConfig != nil { + opts = append(opts, remote.WithAuth(authn.FromConfig(*authConfig))) + } + + if err := remote.Write(dstRef, img, opts...); err != nil { + return nil, fmt.Errorf("push to local registry: %w", wrapRegistryError(err)) + } + + return &MirrorResult{ + SourceImage: req.SourceImage, + LocalRef: localRef, + Digest: digest.String(), + }, nil +} + +// normalizeToLocalRef converts a source image reference to a normalized local reference. +// It strips the docker.io/ prefix and library/ prefix for cleaner local refs. +// +// Examples: +// - "docker.io/onkernel/nodejs22-base:0.1.1" -> "onkernel/nodejs22-base:0.1.1" +// - "docker.io/library/alpine:3.21" -> "alpine:3.21" +// - "nginx:1.21" -> "nginx:1.21" +// - "gcr.io/google-containers/pause:3.2" -> "gcr.io/google-containers/pause:3.2" +func normalizeToLocalRef(ref name.Reference) string { + // Get the repository name (includes registry for non-Docker Hub images) + repo := ref.Context().String() + + // Strip index.docker.io/ prefix (canonical form of docker.io) + repo = strings.TrimPrefix(repo, "index.docker.io/") + + // Strip docker.io/ prefix + repo = strings.TrimPrefix(repo, "docker.io/") + + // Strip library/ prefix for official images + repo = strings.TrimPrefix(repo, "library/") + + // Build the tag or digest suffix + var suffix string + if tag, ok := ref.(name.Tag); ok { + suffix = ":" + tag.TagStr() + } else if dig, ok := ref.(name.Digest); ok { + suffix = "@" + dig.DigestStr() + } + + return repo + suffix +} + +// stripScheme removes http:// or https:// prefix from a URL +func stripScheme(url string) string { + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + return url +} diff --git a/lib/images/mirror_test.go b/lib/images/mirror_test.go new file mode 100644 index 0000000..1fc2613 --- /dev/null +++ b/lib/images/mirror_test.go @@ -0,0 +1,91 @@ +package images + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeToLocalRef(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "docker hub user image with tag", + input: "docker.io/onkernel/nodejs22-base:0.1.1", + expected: "onkernel/nodejs22-base:0.1.1", + }, + { + name: "docker hub user image without registry prefix", + input: "onkernel/nodejs22-base:0.1.1", + expected: "onkernel/nodejs22-base:0.1.1", + }, + { + name: "docker hub official image with tag", + input: "docker.io/library/alpine:3.21", + expected: "alpine:3.21", + }, + { + name: "docker hub official image short form", + input: "alpine:3.21", + expected: "alpine:3.21", + }, + { + name: "docker hub image with index.docker.io", + input: "index.docker.io/onkernel/nodejs22-base:0.1.1", + expected: "onkernel/nodejs22-base:0.1.1", + }, + { + name: "gcr.io image", + input: "gcr.io/google-containers/pause:3.2", + expected: "gcr.io/google-containers/pause:3.2", + }, + { + name: "ghcr.io image", + input: "ghcr.io/some-org/some-image:v1.0", + expected: "ghcr.io/some-org/some-image:v1.0", + }, + { + name: "image with latest tag", + input: "nginx:latest", + expected: "nginx:latest", + }, + { + name: "image without tag uses latest", + input: "nginx", + expected: "nginx:latest", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref, err := name.ParseReference(tt.input) + require.NoError(t, err) + result := normalizeToLocalRef(ref) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestStripScheme(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"https://localhost:8080", "localhost:8080"}, + {"http://localhost:8080", "localhost:8080"}, + {"localhost:8080", "localhost:8080"}, + {"https://registry.example.com", "registry.example.com"}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := stripScheme(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/lib/registry/registry.go b/lib/registry/registry.go index 44535f4..651baf9 100644 --- a/lib/registry/registry.go +++ b/lib/registry/registry.go @@ -138,7 +138,16 @@ func (w *responseWrapper) WriteHeader(code int) { } // triggerConversion queues the image for conversion to ext4 disk format. +// Skips BuildKit cache images (cache/*) since they're not runnable containers. func (r *Registry) triggerConversion(repo, reference, dockerDigest string) { + // Skip BuildKit cache images - they use a custom mediatype that can't be + // unpacked as a standard OCI image. BuildKit imports them directly from + // the registry without needing local conversion. + // Note: repo may include host prefix (e.g., "10.102.0.1:8083/cache/global/node") + if strings.HasPrefix(repo, "cache/") || strings.Contains(repo, "/cache/") { + return + } + imageRef := repo + ":" + reference if strings.HasPrefix(reference, "sha256:") { imageRef = repo + "@" + reference