branch:
installer.ts
12736 bytesRaw
/**
* NPM package installer for virtual file systems.
*
* This module fetches packages from the npm registry and populates
* a virtual node_modules directory structure.
*/
import * as semver from "semver";
import type { Files } from "./types";
const NPM_REGISTRY = "https://registry.npmjs.org";
const DEFAULT_TIMEOUT_MS = 30000; // 30 seconds
/**
* Fetch with a timeout.
* Throws an error if the request takes longer than the specified timeout.
*/
async function fetchWithTimeout(
url: string,
options: RequestInit = {},
timeoutMs = DEFAULT_TIMEOUT_MS
): Promise<Response> {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
try {
return await fetch(url, { ...options, signal: controller.signal });
} catch (error) {
if (error instanceof Error && error.name === "AbortError") {
throw new Error(`Request timed out after ${timeoutMs}ms: ${url}`);
}
throw error;
} finally {
clearTimeout(timeoutId);
}
}
interface PackageJson {
name: string;
version: string;
main?: string;
module?: string;
exports?: unknown;
dependencies?: Record<string, string>;
peerDependencies?: Record<string, string>;
devDependencies?: Record<string, string>;
dist?: {
tarball: string;
integrity?: string;
};
}
interface NpmPackageMetadata {
name: string;
"dist-tags": Record<string, string>;
versions: Record<string, PackageJson>;
}
interface InstallOptions {
/**
* Include devDependencies (default: false)
*/
dev?: boolean;
/**
* Registry URL (default: https://registry.npmjs.org)
*/
registry?: string;
}
interface InstallResult {
/**
* Files with node_modules populated
*/
files: Files;
/**
* Packages that were installed
*/
installed: string[];
/**
* Warnings encountered during installation
*/
warnings: string[];
}
/**
* Install npm dependencies into a virtual file system.
*
* Reads the package.json from the files, resolves all dependencies,
* and populates node_modules with the package contents.
*
* @param files - Virtual file system containing package.json
* @param options - Installation options
* @returns Files with node_modules populated
*/
export async function installDependencies(
files: Files,
options: InstallOptions = {}
): Promise<InstallResult> {
const { dev = false, registry = NPM_REGISTRY } = options;
const result: InstallResult = {
files: { ...files },
installed: [],
warnings: []
};
// Read package.json
const packageJsonContent = files["package.json"];
if (!packageJsonContent) {
return result; // No package.json, nothing to install
}
let packageJson: PackageJson;
try {
packageJson = JSON.parse(packageJsonContent) as PackageJson;
} catch {
result.warnings.push("Failed to parse package.json");
return result;
}
// Collect dependencies to install
const depsToInstall: Record<string, string> = {
...packageJson.dependencies,
...(dev ? packageJson.devDependencies : {})
};
if (Object.keys(depsToInstall).length === 0) {
return result; // No dependencies to install
}
// Track installed packages to avoid duplicates
const installedPackages = new Map<string, string>(); // name -> version
// Track in-progress installations to avoid duplicate work
const inProgress = new Map<string, Promise<void>>();
// Install all dependencies in parallel
await Promise.all(
Object.entries(depsToInstall).map(([name, versionRange]) =>
installPackage(
name,
versionRange,
result,
installedPackages,
inProgress,
registry
)
)
);
return result;
}
/**
* Install a single package and its dependencies recursively.
*/
async function installPackage(
name: string,
versionRange: string,
result: InstallResult,
installedPackages: Map<string, string>,
inProgress: Map<string, Promise<void>>,
registry: string
): Promise<void> {
// Skip if already installed
if (installedPackages.has(name)) {
return;
}
// If installation is already in progress, wait for it
const existing = inProgress.get(name);
if (existing) {
return existing;
}
// Create the installation promise
const installPromise = (async () => {
try {
// Fetch package metadata from registry
const metadata = await fetchPackageMetadata(name, registry);
// Resolve version from range
const version = resolveVersion(versionRange, metadata);
if (!version) {
result.warnings.push(
`Could not resolve version for ${name}@${versionRange}`
);
return;
}
// Get the specific version metadata
const versionMetadata = metadata.versions[version];
if (!versionMetadata) {
result.warnings.push(`Version ${version} not found for ${name}`);
return;
}
// Mark as installed (before fetching to prevent cycles)
installedPackages.set(name, version);
result.installed.push(`${name}@${version}`);
// Fetch and extract the package tarball
const packageFiles = await fetchPackageFiles(name, versionMetadata);
// Add files to node_modules
for (const [filePath, content] of Object.entries(packageFiles)) {
result.files[`node_modules/${name}/${filePath}`] = content;
}
// Install dependencies in parallel
const deps = versionMetadata.dependencies ?? {};
await Promise.all(
Object.entries(deps).map(([depName, depVersion]) =>
installPackage(
depName,
depVersion,
result,
installedPackages,
inProgress,
registry
)
)
);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
result.warnings.push(`Failed to install ${name}: ${message}`);
}
})();
// Track in progress
inProgress.set(name, installPromise);
try {
await installPromise;
} finally {
inProgress.delete(name);
}
}
/**
* Fetch package metadata from npm registry.
*/
async function fetchPackageMetadata(
name: string,
registry: string
): Promise<NpmPackageMetadata> {
// Handle scoped packages
const encodedName = name.startsWith("@")
? `@${encodeURIComponent(name.slice(1))}`
: name;
const url = `${registry}/${encodedName}`;
const response = await fetchWithTimeout(url, {
headers: {
// Use abbreviated metadata to avoid fetching megabytes of version data
Accept:
"application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8"
}
});
if (!response.ok) {
throw new Error(`Failed to fetch package metadata: ${response.status}`);
}
return (await response.json()) as NpmPackageMetadata;
}
/**
* Resolve a semver range to a specific version.
*/
function resolveVersion(
range: string,
metadata: NpmPackageMetadata
): string | undefined {
// Handle special cases
if (range === "latest" || range === "*") {
return metadata["dist-tags"]["latest"];
}
// Handle exact versions
if (metadata.versions[range]) {
return range;
}
// Handle dist-tags (e.g., "next", "beta")
if (metadata["dist-tags"][range]) {
return metadata["dist-tags"][range];
}
// Use semver.maxSatisfying to find the best matching version
const versions = Object.keys(metadata.versions);
const match = semver.maxSatisfying(versions, range);
return match ?? undefined;
}
/**
* Fetch and extract package files from npm tarball.
*/
async function fetchPackageFiles(
name: string,
metadata: PackageJson
): Promise<Record<string, string>> {
const tarballUrl = metadata.dist?.tarball;
if (!tarballUrl) {
throw new Error(`No tarball URL for ${name}`);
}
// Fetch the tarball (use longer timeout for potentially large packages)
const response = await fetchWithTimeout(
tarballUrl,
{},
DEFAULT_TIMEOUT_MS * 2
);
if (!response.ok) {
throw new Error(`Failed to fetch tarball: ${response.status}`);
}
// Get the tarball as array buffer
const buffer = await response.arrayBuffer();
// Extract the tarball (npm tarballs are gzipped tar files)
return extractTarball(new Uint8Array(buffer));
}
/**
* Extract files from a gzipped tarball.
*
* npm packages are distributed as .tgz files (gzipped tar).
* The contents are in a "package/" directory.
*/
async function extractTarball(
data: Uint8Array
): Promise<Record<string, string>> {
// Decompress gzip
const decompressed = await decompress(data);
// Parse tar
return parseTar(decompressed);
}
/**
* Decompress gzip data using DecompressionStream.
*/
async function decompress(data: Uint8Array): Promise<Uint8Array> {
// Use DecompressionStream (available in Workers and modern browsers)
const ds = new DecompressionStream("gzip");
const writer = ds.writable.getWriter();
const reader = ds.readable.getReader();
// Write compressed data
writer.write(data as Uint8Array<ArrayBuffer>).catch(() => {});
writer.close().catch(() => {});
// Read decompressed data
const chunks: Uint8Array[] = [];
let totalLength = 0;
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
totalLength += value.length;
}
// Concatenate chunks
const result = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
result.set(chunk, offset);
offset += chunk.length;
}
return result;
}
/**
* Parse a tar archive and extract text files.
*
* TAR format:
* - 512-byte header blocks
* - File content (padded to 512 bytes)
* - Two empty blocks at the end
*/
function parseTar(data: Uint8Array): Record<string, string> {
const files: Record<string, string> = {};
const textDecoder = new TextDecoder();
let offset = 0;
while (offset < data.length - 512) {
// Read header
const header = data.slice(offset, offset + 512);
// Check for empty block (end of archive)
if (header.every((b) => b === 0)) {
break;
}
// Parse header fields
const name = readString(header, 0, 100);
const sizeStr = readString(header, 124, 12);
const typeFlag = header[156];
// Parse size (octal)
const size = parseInt(sizeStr.trim(), 8) || 0;
// Move past header
offset += 512;
// Only process regular files (type '0' or '\0')
if ((typeFlag === 48 || typeFlag === 0) && size > 0) {
// Read file content
const content = data.slice(offset, offset + size);
// Remove "package/" prefix from npm tarballs
let filePath = name;
if (filePath.startsWith("package/")) {
filePath = filePath.slice(8);
}
// Only include text files (skip binary files)
if (isTextFile(filePath)) {
try {
files[filePath] = textDecoder.decode(content);
} catch {
// Skip files that can't be decoded as text
}
}
}
// Move to next block (content is padded to 512 bytes)
offset += Math.ceil(size / 512) * 512;
}
return files;
}
/**
* Read a null-terminated string from a buffer.
*/
function readString(
buffer: Uint8Array,
offset: number,
length: number
): string {
const bytes = buffer.slice(offset, offset + length);
const nullIndex = bytes.indexOf(0);
const relevantBytes = nullIndex >= 0 ? bytes.slice(0, nullIndex) : bytes;
return new TextDecoder().decode(relevantBytes);
}
/**
* Check if a file path is likely a text file.
*/
function isTextFile(path: string): boolean {
const textExtensions = [
".js",
".mjs",
".cjs",
".ts",
".mts",
".cts",
".tsx",
".jsx",
".json",
".md",
".txt",
".css",
".html",
".yml",
".yaml",
".toml",
".xml",
".svg",
".map",
".d.ts",
".d.mts",
".d.cts"
];
// Check common config files without extensions
const configFiles = [
"LICENSE",
"README",
"CHANGELOG",
"package.json",
"tsconfig.json",
".npmignore",
".gitignore"
];
const fileName = path.split("/").pop() ?? "";
if (
configFiles.some((f) => fileName.toUpperCase().startsWith(f.toUpperCase()))
) {
return true;
}
return textExtensions.some((ext) => path.toLowerCase().endsWith(ext));
}
/**
* Check if files contain a package.json with dependencies that need installing.
*/
export function hasDependencies(files: Files): boolean {
const packageJson = files["package.json"];
if (!packageJson) return false;
try {
const pkg = JSON.parse(packageJson);
const deps = pkg.dependencies ?? {};
return Object.keys(deps).length > 0;
} catch {
return false;
}
}