Skip to content

[rush] Remove node tar fallback #3653

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
David Michon (dmichon-msft) merged 1 commit into from
Sep 27, 2022
Merged

[rush] Remove node tar fallback #3653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions common/changes/@microsoft/rush/node-tar-cleanup_2022-09-26-20-33.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "Remove fallback from tar binary to npm 'tar' package. The npm 'tar' package would sometimes produce invalid output if the cache entry was corrupt.",
"type": "none"
}
],
"packageName": "@microsoft/rush"
}
104 changes: 19 additions & 85 deletions libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,8 @@
// See LICENSE in the project root for license information.

import * as path from 'path';
import events from 'events';
import * as crypto from 'crypto';
import type * as stream from 'stream';
import * as tar from 'tar';
import { FileSystem, Path, ITerminal, FolderItem } from '@rushstack/node-core-library';
import { FileSystem, Path, ITerminal, FolderItem, InternalError } from '@rushstack/node-core-library';

import { RushConfigurationProject } from '../../api/RushConfigurationProject';
import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer';
Expand All @@ -16,7 +13,7 @@ import { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration';
import { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider';
import { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider';
import { TarExecutable } from '../../utilities/TarExecutable';
import { Utilities } from '../../utilities/Utilities';
import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration';

export interface IProjectBuildCacheOptions {
buildCacheConfiguration: BuildCacheConfiguration;
Expand Down Expand Up @@ -194,47 +191,12 @@ export class ProjectBuildCache {
});
if (tarExitCode === 0) {
restoreSuccess = true;
terminal.writeLine('Successfully restored output from the build cache.');
} else {
terminal.writeWarningLine(
`"tar" exited with code ${tarExitCode} while attempting to restore cache entry. ` +
'Rush will attempt to extract from the cache entry with a JavaScript implementation of tar. ' +
`See "${logFilePath}" for logs from the tar process.`
);
}
}

if (!restoreSuccess) {
if (!cacheEntryBuffer && localCacheEntryPath) {
cacheEntryBuffer = await FileSystem.readFileToBufferAsync(localCacheEntryPath);
}

if (!cacheEntryBuffer) {
throw new Error('Expected the cache entry buffer to be set.');
}

// If we don't have tar on the PATH, if we failed to update the local cache entry,
// or if the tar binary failed, untar in-memory
const tarStream: stream.Writable = tar.extract({
cwd: projectFolderPath,
// Set to true to omit writing mtime value for extracted entries.
m: true
});
try {
const tarPromise: Promise<unknown> = events.once(tarStream, 'drain');
tarStream.write(cacheEntryBuffer);
await tarPromise;
restoreSuccess = true;
} catch (e) {
restoreSuccess = false;
terminal.writeWarningLine('Unable to restore output from the build cache.');
}
}

if (restoreSuccess) {
terminal.writeLine('Successfully restored output from the build cache.');
} else {
terminal.writeWarningLine('Unable to restore output from the build cache.');
}

if (updateLocalCacheSuccess === false) {
terminal.writeWarningLine('Unable to update the local build cache with data from the cloud cache.');
}
Expand All @@ -254,7 +216,6 @@ export class ProjectBuildCache {
return false;
}

const projectFolderPath: string = this._project.projectFolder;
const filesToCache: IPathsToCache | undefined = await this._tryCollectPathsToCacheAsync(terminal);
if (!filesToCache) {
return false;
Expand Down Expand Up @@ -290,34 +251,19 @@ export class ProjectBuildCache {
} else {
terminal.writeWarningLine(
`"tar" exited with code ${tarExitCode} while attempting to create the cache entry. ` +
'Rush will attempt to create the cache entry with a JavaScript implementation of tar. ' +
`See "${logFilePath}" for logs from the tar process.`
);
return false;
}
} else {
terminal.writeWarningLine(
`Unable to locate "tar". Please ensure that "tar" is on your PATH environment variable, or set the ` +
`${EnvironmentVariableNames.RUSH_TAR_BINARY_PATH} environment variable to the full path to the "tar" binary.`
);
return false;
}

let cacheEntryBuffer: Buffer | undefined;
let setLocalCacheEntryPromise: Promise<string> | undefined;
if (!localCacheEntryPath) {
// If we weren't able to create the cache entry with tar, try to do it with the "tar" NPM package
const tarStream: stream.Readable = tar.create(
{
gzip: true,
portable: true,
strict: true,
cwd: projectFolderPath
},
filesToCache.outputFilePaths
);
cacheEntryBuffer = await Utilities.readStreamToBufferAsync(tarStream);
setLocalCacheEntryPromise = this._localBuildCacheProvider.trySetCacheEntryBufferAsync(
terminal,
cacheId,
cacheEntryBuffer
);
} else {
setLocalCacheEntryPromise = Promise.resolve(localCacheEntryPath);
}

let setCloudCacheEntryPromise: Promise<boolean> | undefined;

Expand All @@ -326,12 +272,10 @@ export class ProjectBuildCache {
// write to the local build cache.

if (this._cloudBuildCacheProvider?.isCacheWriteAllowed) {
if (!cacheEntryBuffer) {
if (localCacheEntryPath) {
cacheEntryBuffer = await FileSystem.readFileToBufferAsync(localCacheEntryPath);
} else {
throw new Error('Expected the local cache entry path to be set.');
}
if (localCacheEntryPath) {
cacheEntryBuffer = await FileSystem.readFileToBufferAsync(localCacheEntryPath);
} else {
throw new InternalError('Expected the local cache entry path to be set.');
}

setCloudCacheEntryPromise = this._cloudBuildCacheProvider?.trySetCacheEntryBufferAsync(
Expand All @@ -341,24 +285,14 @@ export class ProjectBuildCache {
);
}

let localCachePath: string;
let updateCloudCacheSuccess: boolean;
if (setCloudCacheEntryPromise) {
[updateCloudCacheSuccess, localCachePath] = await Promise.all([
setCloudCacheEntryPromise,
setLocalCacheEntryPromise
]);
} else {
updateCloudCacheSuccess = true;
localCachePath = await setLocalCacheEntryPromise;
}
const updateCloudCacheSuccess: boolean | undefined = (await setCloudCacheEntryPromise) ?? true;

const success: boolean = updateCloudCacheSuccess && !!localCachePath;
const success: boolean = updateCloudCacheSuccess && !!localCacheEntryPath;
if (success) {
terminal.writeLine('Successfully set cache entry.');
} else if (!localCachePath && updateCloudCacheSuccess) {
} else if (!localCacheEntryPath && updateCloudCacheSuccess) {
terminal.writeWarningLine('Unable to set local cache entry.');
} else if (localCachePath && !updateCloudCacheSuccess) {
} else if (localCacheEntryPath && !updateCloudCacheSuccess) {
terminal.writeWarningLine('Unable to set cloud cache entry.');
} else {
terminal.writeWarningLine('Unable to set both cloud and local cache entries.');
Expand Down