Skip to content

add --build-output-folder #1309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Patrick Stöckle (pstoeckle) wants to merge 1 commit into
base: main
Choose a base branch
from
Open

add --build-output-folder #1309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 38 additions & 10 deletions src/spec-node/containerFeatures.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,22 @@ export const getSafeId = (str: string) => str
.replace(/^[\d_]+/g, '_')
.toUpperCase();

export async function extendImage(params: DockerResolverParameters, config: SubstitutedConfig<DevContainerConfig>, imageName: string, additionalImageNames: string[], additionalFeatures: Record<string, string | boolean | Record<string, string | boolean>>, canAddLabelsToContainer: boolean) {
export interface PreparedBuildInfo {
dockerfile: string;
context: string;
target: string | undefined;
buildArgs: Record<string, string>;
buildContexts: Record<string, string>;
securityOpts: string[];
imageNames: string[];
}

export async function extendImage(params: DockerResolverParameters, config: SubstitutedConfig<DevContainerConfig>, imageName: string, additionalImageNames: string[], additionalFeatures: Record<string, string | boolean | Record<string, string | boolean>>, canAddLabelsToContainer: boolean, buildOutputFolder?: string) {
const { common } = params;
const { cliHost, output } = common;

const imageBuildInfo = await getImageBuildInfoFromImage(params, imageName, config.substitute);
const extendImageDetails = await getExtendImageBuildInfo(params, config, imageName, imageBuildInfo, undefined, additionalFeatures, canAddLabelsToContainer);
const extendImageDetails = await getExtendImageBuildInfo(params, config, imageName, imageBuildInfo, undefined, additionalFeatures, canAddLabelsToContainer, buildOutputFolder);
if (!extendImageDetails?.featureBuildInfo) {
// no feature extensions - return
if (additionalImageNames.length) {
Expand All @@ -53,10 +63,27 @@ export async function extendImage(params: DockerResolverParameters, config: Subs
const { featureBuildInfo, featuresConfig } = extendImageDetails;

// Got feature extensions -> build the image
const dockerfilePath = cliHost.path.join(featureBuildInfo.dstFolder, 'Dockerfile.extended');
const dockerfilePath = cliHost.path.join(featureBuildInfo.dstFolder, buildOutputFolder ? 'Dockerfile' : 'Dockerfile.extended');
await cliHost.writeFile(dockerfilePath, Buffer.from(featureBuildInfo.dockerfilePrefixContent + featureBuildInfo.dockerfileContent));
const folderImageName = getFolderImageName(common);
const updatedImageName = `${imageName.startsWith(folderImageName) ? imageName : folderImageName}-features`;
const outputImageNames = additionalImageNames.length > 0 ? additionalImageNames : [updatedImageName];
if (buildOutputFolder) {
return {
updatedImageName: outputImageNames,
imageMetadata: getDevcontainerMetadata(imageBuildInfo.metadata, config, featuresConfig),
imageDetails: async () => imageBuildInfo.imageDetails,
preparedBuild: {
dockerfile: dockerfilePath,
context: buildOutputFolder,
target: featureBuildInfo.overrideTarget,
buildArgs: featureBuildInfo.buildArgs,
buildContexts: featureBuildInfo.buildKitContexts,
securityOpts: featureBuildInfo.securityOpts,
imageNames: outputImageNames,
} satisfies PreparedBuildInfo,
};
}

const args: string[] = [];
if (!params.buildKitVersion &&
Expand Down Expand Up @@ -132,16 +159,17 @@ export async function extendImage(params: DockerResolverParameters, config: Subs
await dockerCLI(infoParams, ...args);
}
return {
updatedImageName: additionalImageNames.length > 0 ? additionalImageNames : [updatedImageName],
updatedImageName: outputImageNames,
imageMetadata: getDevcontainerMetadata(imageBuildInfo.metadata, config, featuresConfig),
imageDetails: async () => imageBuildInfo.imageDetails,
};
}

export async function getExtendImageBuildInfo(params: DockerResolverParameters, config: SubstitutedConfig<DevContainerConfig>, baseName: string, imageBuildInfo: ImageBuildInfo, composeServiceUser: string | undefined, additionalFeatures: Record<string, string | boolean | Record<string, string | boolean>>, canAddLabelsToContainer: boolean): Promise<{ featureBuildInfo?: ImageBuildOptions; featuresConfig?: FeaturesConfig; labels?: Record<string, string> } | undefined> {
export async function getExtendImageBuildInfo(params: DockerResolverParameters, config: SubstitutedConfig<DevContainerConfig>, baseName: string, imageBuildInfo: ImageBuildInfo, composeServiceUser: string | undefined, additionalFeatures: Record<string, string | boolean | Record<string, string | boolean>>, canAddLabelsToContainer: boolean, buildOutputFolder?: string): Promise<{ featureBuildInfo?: ImageBuildOptions; featuresConfig?: FeaturesConfig; labels?: Record<string, string> } | undefined> {

// Creates the folder where the working files will be setup.
const dstFolder = await createFeaturesTempFolder(params.common);
const dstFolder = buildOutputFolder || await createFeaturesTempFolder(params.common);
await params.common.cliHost.mkdirp(dstFolder);

// Processes the user's configuration.
const platform = params.common.cliHost.platform;
Expand All @@ -161,7 +189,7 @@ export async function getExtendImageBuildInfo(params: DockerResolverParameters,
}

// Generates the end configuration.
const featureBuildInfo = await getFeaturesBuildOptions(params, config, featuresConfig, baseName, imageBuildInfo, composeServiceUser);
const featureBuildInfo = await getFeaturesBuildOptions(params, config, featuresConfig, baseName, imageBuildInfo, composeServiceUser, !!buildOutputFolder);
if (!featureBuildInfo) {
return undefined;
}
Expand Down Expand Up @@ -224,7 +252,7 @@ function getOmitDevcontainerPropertyOverride(resolverParams: { omitConfigRemotEn
return [];
}

async function getFeaturesBuildOptions(params: DockerResolverParameters, devContainerConfig: SubstitutedConfig<DevContainerConfig>, featuresConfig: FeaturesConfig, baseName: string, imageBuildInfo: ImageBuildInfo, composeServiceUser: string | undefined): Promise<ImageBuildOptions | undefined> {
async function getFeaturesBuildOptions(params: DockerResolverParameters, devContainerConfig: SubstitutedConfig<DevContainerConfig>, featuresConfig: FeaturesConfig, baseName: string, imageBuildInfo: ImageBuildInfo, composeServiceUser: string | undefined, prepareOnly: boolean): Promise<ImageBuildOptions | undefined> {
const { common } = params;
const { cliHost, output } = common;
const { dstFolder } = featuresConfig;
Expand All @@ -241,7 +269,7 @@ async function getFeaturesBuildOptions(params: DockerResolverParameters, devCont
// TODO generate an image name that is specific to this dev container?
const buildKitVersionParsed = params.buildKitVersion?.versionMatch ? parseVersion(params.buildKitVersion.versionMatch) : undefined;
const minRequiredVersion = [0, 8, 0];
const useBuildKitBuildContexts = buildKitVersionParsed ? !isEarlierVersion(buildKitVersionParsed, minRequiredVersion) : false;
const useBuildKitBuildContexts = prepareOnly || (buildKitVersionParsed ? !isEarlierVersion(buildKitVersionParsed, minRequiredVersion) : false);
const buildContentImageName = 'dev_container_feature_content_temp';
const disableSELinuxLabels = useBuildKitBuildContexts && await isUsingSELinuxLabels(params);
// Access Docker engine version
Expand Down Expand Up @@ -378,7 +406,7 @@ async function isUsingSELinuxLabels(params: DockerResolverParameters): Promise<b
} catch {
// If we can't run the commands, assume SELinux is not enabled.
return false;

}
}

Expand Down
42 changes: 37 additions & 5 deletions src/spec-node/devContainersSpecCLI.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,7 @@ function buildOptions(y: Argv) {
'push': { type: 'boolean', default: false, description: 'Push to a container registry.' },
'label': { type: 'string', description: 'Provide key and value configuration that adds metadata to an image' },
'output': { type: 'string', description: 'Overrides the default behavior to load built images into the local docker registry. Valid options are the same ones provided to the --output option of docker buildx build.' },
'build-output-folder': { type: 'string', description: 'Prepare the Dockerfile, Features, and build metadata in this empty folder without building the image.' },
'additional-features': { type: 'string', description: 'Additional features to apply to the dev container (JSON as per "features" section in devcontainer.json)' },
'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' },
'skip-persisting-customizations-from-features': { type: 'boolean', default: false, hidden: true, description: 'Do not save customizations from referenced Features as image metadata' },
Expand All @@ -588,6 +589,9 @@ function buildOptions(y: Argv) {
'omit-syntax-directive': { type: 'boolean', default: false, hidden: true, description: 'Omit Dockerfile syntax directives' },
})
.check(argv => {
if (argv['build-output-folder'] && (argv.push || argv.output)) {
throw new Error('--build-output-folder cannot be used with --push or --output.');
}
if (argv['no-lockfile'] && argv['frozen-lockfile']) {
throw new Error('--no-lockfile and --frozen-lockfile are mutually exclusive.');
}
Expand Down Expand Up @@ -633,6 +637,7 @@ async function doBuild({
'push': buildxPush,
'label': buildxLabel,
'output': buildxOutput,
'build-output-folder': buildOutputFolderArg,
'cache-to': buildxCacheTo,
'additional-features': additionalFeaturesJson,
'skip-feature-auto-mapping': skipFeatureAutoMapping,
Expand All @@ -654,6 +659,7 @@ async function doBuild({
};
try {
const workspaceFolder = workspaceFolderArg ? path.resolve(process.cwd(), workspaceFolderArg) : process.cwd();
const buildOutputFolder = buildOutputFolderArg ? path.resolve(process.cwd(), buildOutputFolderArg) : undefined;
const configFile: URI | undefined = configParam ? URI.file(path.resolve(process.cwd(), configParam)) : undefined;
const overrideConfigFile: URI | undefined = /* overrideConfig ? URI.file(path.resolve(process.cwd(), overrideConfig)) : */ undefined;
const addCacheFroms = addCacheFrom ? (Array.isArray(addCacheFrom) ? addCacheFrom as string[] : [addCacheFrom]) : [];
Expand Down Expand Up @@ -703,6 +709,12 @@ async function doBuild({

const { common, dockerComposeCLI } = params;
const { cliHost, env, output } = common;
if (buildOutputFolder) {
if (await cliHost.isFolder(buildOutputFolder) && (await cliHost.readDir(buildOutputFolder)).length) {
throw new ContainerError({ description: `Build output folder (${buildOutputFolder}) must be empty.` });
}
await cliHost.mkdirp(buildOutputFolder);
}
const workspace = workspaceFromPath(cliHost.path, workspaceFolder);
const configPath = configFile ? configFile : workspace
? (await getDevContainerConfigPathIn(cliHost, workspace.configFolderPath)
Expand All @@ -715,6 +727,7 @@ async function doBuild({
const configWithRaw = configs.config;
const { config } = configWithRaw;
let imageNameResult: string[] = [''];
let preparedBuild;

if (buildxOutput && buildxPush) {
throw new ContainerError({ description: '--push true cannot be used with --output.' });
Expand All @@ -732,7 +745,9 @@ async function doBuild({
if (isDockerFileConfig(config)) {

// Build the base image and extend with features etc.
let { updatedImageName } = await buildNamedImageAndExtend(params, configWithRaw as SubstitutedConfig<DevContainerFromDockerfileConfig>, additionalFeatures, false, imageNames);
const buildResult = await buildNamedImageAndExtend(params, configWithRaw as SubstitutedConfig<DevContainerFromDockerfileConfig>, additionalFeatures, false, imageNames, buildOutputFolder);
const { updatedImageName } = buildResult;
preparedBuild = buildResult.preparedBuild;

if (imageNames) {
imageNameResult = imageNames;
Expand Down Expand Up @@ -762,7 +777,7 @@ async function doBuild({
if (envFile) {
composeGlobalArgs.push('--env-file', envFile);
}

const composeConfig = await readDockerComposeConfig(buildParams, composeFiles, envFile);
const projectName = await getProjectName(params, workspace, composeFiles, composeConfig);
const services = Object.keys(composeConfig.services || {});
Expand All @@ -772,14 +787,21 @@ async function doBuild({

const versionPrefix = await readVersionPrefix(cliHost, composeFiles);
const infoParams = { ...params, common: { ...params.common, output: makeLog(buildParams.output, LogLevel.Info) } };
const { overrideImageName } = await buildAndExtendDockerCompose(configWithRaw as SubstitutedConfig<DevContainerFromDockerComposeConfig>, projectName, infoParams, composeFiles, envFile, composeGlobalArgs, [config.service], params.buildNoCache || false, params.common.persistedFolder, 'docker-compose.devcontainer.build', versionPrefix, additionalFeatures, false, addCacheFroms);
const buildResult = await buildAndExtendDockerCompose(configWithRaw as SubstitutedConfig<DevContainerFromDockerComposeConfig>, projectName, infoParams, composeFiles, envFile, composeGlobalArgs, [config.service], params.buildNoCache || false, buildOutputFolder || params.common.persistedFolder, 'docker-compose.devcontainer.build', versionPrefix, additionalFeatures, false, addCacheFroms, !!buildOutputFolder, buildOutputFolder);
const { overrideImageName } = buildResult;
preparedBuild = buildResult.preparedBuild;
if (preparedBuild && imageNames) {
preparedBuild.imageNames = imageNames;
}

const service = composeConfig.services[config.service];
const originalImageName = overrideImageName || service.image || getDefaultImageName(await buildParams.dockerComposeCLI(), projectName, config.service);

if (imageNames) {
// Future improvement: Compose 2.6.0 (released 2022-05-30) added `tags` to the compose file.
if (params.isTTY) {
if (buildOutputFolder) {
// The requested names are recorded in build-output.json for the external builder.
} else if (params.isTTY) {
await Promise.all(imageNames.map(imageName => dockerPtyCLI(params, 'tag', originalImageName, imageName)));
} else {
await Promise.all(imageNames.map(imageName => dockerCLI(params, 'tag', originalImageName, imageName)));
Expand All @@ -795,7 +817,9 @@ async function doBuild({
}

await inspectDockerImage(params, config.image, true);
const { updatedImageName } = await extendImage(params, configWithRaw, config.image, imageNames || [], additionalFeatures, false);
const buildResult = await extendImage(params, configWithRaw, config.image, imageNames || [], additionalFeatures, false, buildOutputFolder);
const { updatedImageName } = buildResult;
preparedBuild = buildResult.preparedBuild;

if (imageNames) {
imageNameResult = imageNames;
Expand All @@ -804,9 +828,17 @@ async function doBuild({
}
}

if (buildOutputFolder) {
if (!preparedBuild) {
throw new ContainerError({ description: 'Failed to prepare build output.' });
}
await cliHost.writeFile(cliHost.path.join(buildOutputFolder, 'build-output.json'), Buffer.from(JSON.stringify(preparedBuild, undefined, 2) + '\n'));
}

return {
outcome: 'success' as 'success',
imageName: imageNameResult,
buildOutputFolder,
ociAuthDiagnostics: params.common.ociAuthDiagnostics,
dispose,
};
Expand Down
24 changes: 20 additions & 4 deletions src/spec-node/dockerCompose.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { equalPaths, parseVersion, isEarlierVersion, CLIHost } from '../spec-com
import { ContainerDetails, inspectContainer, listContainers, DockerCLIParameters, dockerComposeCLI, dockerComposePtyCLI, PartialExecParameters, DockerComposeCLI, ImageDetails, toExecParameters, toPtyExecParameters, removeContainer, CLIVariant } from '../spec-shutdown/dockerUtils';
import { DevContainerFromDockerComposeConfig, getDockerComposeFilePaths } from '../spec-configuration/configuration';
import { Log, LogLevel, makeLog, terminalEscapeSequences } from '../spec-utils/log';
import { getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeatures';
import { getExtendImageBuildInfo, PreparedBuildInfo, updateRemoteUserUID } from './containerFeatures';
import { Mount, parseMount } from '../spec-configuration/containerFeaturesConfiguration';
import path from 'path';
import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageBuildInfoFromImage, getImageMetadataFromContainer, ImageBuildInfo, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata';
Expand Down Expand Up @@ -149,7 +149,7 @@ export function getBuildInfoForService(composeService: any, cliHostPath: typeof
};
}

export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConfig<DevContainerFromDockerComposeConfig>, projectName: string, params: DockerResolverParameters, localComposeFiles: string[], envFile: string | undefined, composeGlobalArgs: string[], runServices: string[], noCache: boolean, overrideFilePath: string, overrideFilePrefix: string, versionPrefix: string, additionalFeatures: Record<string, string | boolean | Record<string, string | boolean>>, canAddLabelsToContainer: boolean, additionalCacheFroms?: string[], noBuild?: boolean) {
export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConfig<DevContainerFromDockerComposeConfig>, projectName: string, params: DockerResolverParameters, localComposeFiles: string[], envFile: string | undefined, composeGlobalArgs: string[], runServices: string[], noCache: boolean, overrideFilePath: string, overrideFilePrefix: string, versionPrefix: string, additionalFeatures: Record<string, string | boolean | Record<string, string | boolean>>, canAddLabelsToContainer: boolean, additionalCacheFroms?: string[], noBuild?: boolean, buildOutputFolder?: string) {

const { common, dockerCLI, dockerComposeCLI: dockerComposeCLIFunc } = params;
const { cliHost, env, output } = common;
Expand Down Expand Up @@ -190,10 +190,11 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf
const version = parseVersion((await params.dockerComposeCLI()).version);
const supportsAdditionalBuildContexts = params.cliVariant === CLIVariant.Docker && version && !isEarlierVersion(version, [2, 17, 0]);
const optionalBuildKitParams = supportsAdditionalBuildContexts ? params : { ...params, buildKitVersion: undefined };
const extendImageBuildInfo = await getExtendImageBuildInfo(optionalBuildKitParams, configWithRaw, baseName, imageBuildInfo, composeService.user, additionalFeatures, canAddLabelsToContainer);
const extendImageBuildInfo = await getExtendImageBuildInfo(optionalBuildKitParams, configWithRaw, baseName, imageBuildInfo, composeService.user, additionalFeatures, canAddLabelsToContainer, buildOutputFolder);

let overrideImageName: string | undefined;
let buildOverrideContent = '';
let preparedBuild: PreparedBuildInfo | undefined;
if (extendImageBuildInfo?.featureBuildInfo) {
// Avoid retagging a previously pulled image.
if (!serviceInfo.build) {
Expand All @@ -212,8 +213,22 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf
dockerfile = dockerfile.slice(syntaxMatch[0].length);
}
let finalDockerfileContent = `${featureBuildInfo.dockerfilePrefixContent}${dockerfile}\n${featureBuildInfo.dockerfileContent}`;
const finalDockerfilePath = cliHost.path.join(featureBuildInfo?.dstFolder, 'Dockerfile-with-features');
const finalDockerfilePath = cliHost.path.join(featureBuildInfo?.dstFolder, buildOutputFolder ? 'Dockerfile' : 'Dockerfile-with-features');
await cliHost.writeFile(finalDockerfilePath, Buffer.from(finalDockerfileContent));
if (buildOutputFolder) {
preparedBuild = {
dockerfile: finalDockerfilePath,
context: serviceInfo.build?.context || buildOutputFolder,
target: featureBuildInfo.overrideTarget,
buildArgs: {
...(serviceInfo.build?.args || {}),
...featureBuildInfo.buildArgs,
},
buildContexts: featureBuildInfo.buildKitContexts,
securityOpts: featureBuildInfo.securityOpts,
imageNames: overrideImageName ? [overrideImageName] : serviceInfo.image ? [serviceInfo.image] : [],
};
}
buildOverrideContent += ` dockerfile: ${finalDockerfilePath}\n`;
if (serviceInfo.build?.target) {
// Replace target. (Only when set because it is only supported with Docker Compose file version 3.4 and later.)
Expand Down Expand Up @@ -301,6 +316,7 @@ ${cacheFromOverrideContent}
additionalComposeOverrideFiles,
overrideImageName,
labels: extendImageBuildInfo?.labels,
preparedBuild,
};
}

Expand Down
Loading