|
|
// Split chunks line-wise for generators passed to the `std*` options
export const getSplitLinesGenerator = (binary, preserveNewlines, skipped, state) => binary || skipped ? undefined : initializeSplitLines(preserveNewlines, state);
// Same but for synchronous methods
export const splitLinesSync = (chunk, preserveNewlines, objectMode) => objectMode ? chunk.flatMap(item => splitLinesItemSync(item, preserveNewlines)) : splitLinesItemSync(chunk, preserveNewlines);
const splitLinesItemSync = (chunk, preserveNewlines) => { const {transform, final} = initializeSplitLines(preserveNewlines, {}); return [...transform(chunk), ...final()];};
const initializeSplitLines = (preserveNewlines, state) => { state.previousChunks = ''; return { transform: splitGenerator.bind(undefined, state, preserveNewlines), final: linesFinal.bind(undefined, state), };};
// This imperative logic is much faster than using `String.split()` and uses very low memory.
const splitGenerator = function * (state, preserveNewlines, chunk) { if (typeof chunk !== 'string') { yield chunk; return; }
let {previousChunks} = state; let start = -1;
for (let end = 0; end < chunk.length; end += 1) { if (chunk[end] === '\n') { const newlineLength = getNewlineLength(chunk, end, preserveNewlines, state); let line = chunk.slice(start + 1, end + 1 - newlineLength);
if (previousChunks.length > 0) { line = concatString(previousChunks, line); previousChunks = ''; }
yield line; start = end; } }
if (start !== chunk.length - 1) { previousChunks = concatString(previousChunks, chunk.slice(start + 1)); }
state.previousChunks = previousChunks;};
const getNewlineLength = (chunk, end, preserveNewlines, state) => { if (preserveNewlines) { return 0; }
state.isWindowsNewline = end !== 0 && chunk[end - 1] === '\r'; return state.isWindowsNewline ? 2 : 1;};
const linesFinal = function * ({previousChunks}) { if (previousChunks.length > 0) { yield previousChunks; }};
// Unless `preserveNewlines: true` is used, we strip the newline of each line.
// This re-adds them after the user `transform` code has run.
export const getAppendNewlineGenerator = ({binary, preserveNewlines, readableObjectMode, state}) => binary || preserveNewlines || readableObjectMode ? undefined : {transform: appendNewlineGenerator.bind(undefined, state)};
const appendNewlineGenerator = function * ({isWindowsNewline = false}, chunk) { const {unixNewline, windowsNewline, LF, concatBytes} = typeof chunk === 'string' ? linesStringInfo : linesUint8ArrayInfo;
if (chunk.at(-1) === LF) { yield chunk; return; }
const newline = isWindowsNewline ? windowsNewline : unixNewline; yield concatBytes(chunk, newline);};
const concatString = (firstChunk, secondChunk) => `${firstChunk}${secondChunk}`;
const linesStringInfo = { windowsNewline: '\r\n', unixNewline: '\n', LF: '\n', concatBytes: concatString,};
const concatUint8Array = (firstChunk, secondChunk) => { const chunk = new Uint8Array(firstChunk.length + secondChunk.length); chunk.set(firstChunk, 0); chunk.set(secondChunk, firstChunk.length); return chunk;};
const linesUint8ArrayInfo = { windowsNewline: new Uint8Array([0x0D, 0x0A]), unixNewline: new Uint8Array([0x0A]), LF: 0x0A, concatBytes: concatUint8Array,};
|