95977b7d281947c3728a0854033f3f2bbeb981069ee72691277cccd5f9f2564e5663c8531378dd0a54c42fc41a06ccec796a0ea662292ecdd4418d9dd2188c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445
  1. // A readable tar stream creator
  2. // Technically, this is a transform stream that you write paths into,
  3. // and tar format comes out of.
  4. // The `add()` method is like `write()` but returns this,
  5. // and end() return `this` as well, so you can
  6. // do `new Pack(opt).add('files').add('dir').end().pipe(output)
  7. // You could also do something like:
  8. // streamOfPaths().pipe(new Pack()).pipe(new fs.WriteStream('out.tar'))
  9. import fs from 'fs';
  10. import { WriteEntry, WriteEntrySync, WriteEntryTar, } from './write-entry.js';
  11. export class PackJob {
  12. path;
  13. absolute;
  14. entry;
  15. stat;
  16. readdir;
  17. pending = false;
  18. ignore = false;
  19. piped = false;
  20. constructor(path, absolute) {
  21. this.path = path || './';
  22. this.absolute = absolute;
  23. }
  24. }
  25. import { Minipass } from 'minipass';
  26. import * as zlib from 'minizlib';
  27. import { Yallist } from 'yallist';
  28. import { ReadEntry } from './read-entry.js';
  29. import { warnMethod, } from './warn-method.js';
  30. const EOF = Buffer.alloc(1024);
  31. const ONSTAT = Symbol('onStat');
  32. const ENDED = Symbol('ended');
  33. const QUEUE = Symbol('queue');
  34. const CURRENT = Symbol('current');
  35. const PROCESS = Symbol('process');
  36. const PROCESSING = Symbol('processing');
  37. const PROCESSJOB = Symbol('processJob');
  38. const JOBS = Symbol('jobs');
  39. const JOBDONE = Symbol('jobDone');
  40. const ADDFSENTRY = Symbol('addFSEntry');
  41. const ADDTARENTRY = Symbol('addTarEntry');
  42. const STAT = Symbol('stat');
  43. const READDIR = Symbol('readdir');
  44. const ONREADDIR = Symbol('onreaddir');
  45. const PIPE = Symbol('pipe');
  46. const ENTRY = Symbol('entry');
  47. const ENTRYOPT = Symbol('entryOpt');
  48. const WRITEENTRYCLASS = Symbol('writeEntryClass');
  49. const WRITE = Symbol('write');
  50. const ONDRAIN = Symbol('ondrain');
  51. import path from 'path';
  52. import { normalizeWindowsPath } from './normalize-windows-path.js';
  53. export class Pack extends Minipass {
  54. opt;
  55. cwd;
  56. maxReadSize;
  57. preservePaths;
  58. strict;
  59. noPax;
  60. prefix;
  61. linkCache;
  62. statCache;
  63. file;
  64. portable;
  65. zip;
  66. readdirCache;
  67. noDirRecurse;
  68. follow;
  69. noMtime;
  70. mtime;
  71. filter;
  72. jobs;
  73. [WRITEENTRYCLASS];
  74. onWriteEntry;
  75. [QUEUE];
  76. [JOBS] = 0;
  77. [PROCESSING] = false;
  78. [ENDED] = false;
  79. constructor(opt = {}) {
  80. //@ts-ignore
  81. super();
  82. this.opt = opt;
  83. this.file = opt.file || '';
  84. this.cwd = opt.cwd || process.cwd();
  85. this.maxReadSize = opt.maxReadSize;
  86. this.preservePaths = !!opt.preservePaths;
  87. this.strict = !!opt.strict;
  88. this.noPax = !!opt.noPax;
  89. this.prefix = normalizeWindowsPath(opt.prefix || '');
  90. this.linkCache = opt.linkCache || new Map();
  91. this.statCache = opt.statCache || new Map();
  92. this.readdirCache = opt.readdirCache || new Map();
  93. this.onWriteEntry = opt.onWriteEntry;
  94. this[WRITEENTRYCLASS] = WriteEntry;
  95. if (typeof opt.onwarn === 'function') {
  96. this.on('warn', opt.onwarn);
  97. }
  98. this.portable = !!opt.portable;
  99. if (opt.gzip || opt.brotli) {
  100. if (opt.gzip && opt.brotli) {
  101. throw new TypeError('gzip and brotli are mutually exclusive');
  102. }
  103. if (opt.gzip) {
  104. if (typeof opt.gzip !== 'object') {
  105. opt.gzip = {};
  106. }
  107. if (this.portable) {
  108. opt.gzip.portable = true;
  109. }
  110. this.zip = new zlib.Gzip(opt.gzip);
  111. }
  112. if (opt.brotli) {
  113. if (typeof opt.brotli !== 'object') {
  114. opt.brotli = {};
  115. }
  116. this.zip = new zlib.BrotliCompress(opt.brotli);
  117. }
  118. /* c8 ignore next */
  119. if (!this.zip)
  120. throw new Error('impossible');
  121. const zip = this.zip;
  122. zip.on('data', chunk => super.write(chunk));
  123. zip.on('end', () => super.end());
  124. zip.on('drain', () => this[ONDRAIN]());
  125. this.on('resume', () => zip.resume());
  126. }
  127. else {
  128. this.on('drain', this[ONDRAIN]);
  129. }
  130. this.noDirRecurse = !!opt.noDirRecurse;
  131. this.follow = !!opt.follow;
  132. this.noMtime = !!opt.noMtime;
  133. if (opt.mtime)
  134. this.mtime = opt.mtime;
  135. this.filter =
  136. typeof opt.filter === 'function' ? opt.filter : () => true;
  137. this[QUEUE] = new Yallist();
  138. this[JOBS] = 0;
  139. this.jobs = Number(opt.jobs) || 4;
  140. this[PROCESSING] = false;
  141. this[ENDED] = false;
  142. }
  143. [WRITE](chunk) {
  144. return super.write(chunk);
  145. }
  146. add(path) {
  147. this.write(path);
  148. return this;
  149. }
  150. end(path, encoding, cb) {
  151. /* c8 ignore start */
  152. if (typeof path === 'function') {
  153. cb = path;
  154. path = undefined;
  155. }
  156. if (typeof encoding === 'function') {
  157. cb = encoding;
  158. encoding = undefined;
  159. }
  160. /* c8 ignore stop */
  161. if (path) {
  162. this.add(path);
  163. }
  164. this[ENDED] = true;
  165. this[PROCESS]();
  166. /* c8 ignore next */
  167. if (cb)
  168. cb();
  169. return this;
  170. }
  171. write(path) {
  172. if (this[ENDED]) {
  173. throw new Error('write after end');
  174. }
  175. if (path instanceof ReadEntry) {
  176. this[ADDTARENTRY](path);
  177. }
  178. else {
  179. this[ADDFSENTRY](path);
  180. }
  181. return this.flowing;
  182. }
  183. [ADDTARENTRY](p) {
  184. const absolute = normalizeWindowsPath(path.resolve(this.cwd, p.path));
  185. // in this case, we don't have to wait for the stat
  186. if (!this.filter(p.path, p)) {
  187. p.resume();
  188. }
  189. else {
  190. const job = new PackJob(p.path, absolute);
  191. job.entry = new WriteEntryTar(p, this[ENTRYOPT](job));
  192. job.entry.on('end', () => this[JOBDONE](job));
  193. this[JOBS] += 1;
  194. this[QUEUE].push(job);
  195. }
  196. this[PROCESS]();
  197. }
  198. [ADDFSENTRY](p) {
  199. const absolute = normalizeWindowsPath(path.resolve(this.cwd, p));
  200. this[QUEUE].push(new PackJob(p, absolute));
  201. this[PROCESS]();
  202. }
  203. [STAT](job) {
  204. job.pending = true;
  205. this[JOBS] += 1;
  206. const stat = this.follow ? 'stat' : 'lstat';
  207. fs[stat](job.absolute, (er, stat) => {
  208. job.pending = false;
  209. this[JOBS] -= 1;
  210. if (er) {
  211. this.emit('error', er);
  212. }
  213. else {
  214. this[ONSTAT](job, stat);
  215. }
  216. });
  217. }
  218. [ONSTAT](job, stat) {
  219. this.statCache.set(job.absolute, stat);
  220. job.stat = stat;
  221. // now we have the stat, we can filter it.
  222. if (!this.filter(job.path, stat)) {
  223. job.ignore = true;
  224. }
  225. this[PROCESS]();
  226. }
  227. [READDIR](job) {
  228. job.pending = true;
  229. this[JOBS] += 1;
  230. fs.readdir(job.absolute, (er, entries) => {
  231. job.pending = false;
  232. this[JOBS] -= 1;
  233. if (er) {
  234. return this.emit('error', er);
  235. }
  236. this[ONREADDIR](job, entries);
  237. });
  238. }
  239. [ONREADDIR](job, entries) {
  240. this.readdirCache.set(job.absolute, entries);
  241. job.readdir = entries;
  242. this[PROCESS]();
  243. }
  244. [PROCESS]() {
  245. if (this[PROCESSING]) {
  246. return;
  247. }
  248. this[PROCESSING] = true;
  249. for (let w = this[QUEUE].head; !!w && this[JOBS] < this.jobs; w = w.next) {
  250. this[PROCESSJOB](w.value);
  251. if (w.value.ignore) {
  252. const p = w.next;
  253. this[QUEUE].removeNode(w);
  254. w.next = p;
  255. }
  256. }
  257. this[PROCESSING] = false;
  258. if (this[ENDED] && !this[QUEUE].length && this[JOBS] === 0) {
  259. if (this.zip) {
  260. this.zip.end(EOF);
  261. }
  262. else {
  263. super.write(EOF);
  264. super.end();
  265. }
  266. }
  267. }
  268. get [CURRENT]() {
  269. return this[QUEUE] && this[QUEUE].head && this[QUEUE].head.value;
  270. }
  271. [JOBDONE](_job) {
  272. this[QUEUE].shift();
  273. this[JOBS] -= 1;
  274. this[PROCESS]();
  275. }
  276. [PROCESSJOB](job) {
  277. if (job.pending) {
  278. return;
  279. }
  280. if (job.entry) {
  281. if (job === this[CURRENT] && !job.piped) {
  282. this[PIPE](job);
  283. }
  284. return;
  285. }
  286. if (!job.stat) {
  287. const sc = this.statCache.get(job.absolute);
  288. if (sc) {
  289. this[ONSTAT](job, sc);
  290. }
  291. else {
  292. this[STAT](job);
  293. }
  294. }
  295. if (!job.stat) {
  296. return;
  297. }
  298. // filtered out!
  299. if (job.ignore) {
  300. return;
  301. }
  302. if (!this.noDirRecurse &&
  303. job.stat.isDirectory() &&
  304. !job.readdir) {
  305. const rc = this.readdirCache.get(job.absolute);
  306. if (rc) {
  307. this[ONREADDIR](job, rc);
  308. }
  309. else {
  310. this[READDIR](job);
  311. }
  312. if (!job.readdir) {
  313. return;
  314. }
  315. }
  316. // we know it doesn't have an entry, because that got checked above
  317. job.entry = this[ENTRY](job);
  318. if (!job.entry) {
  319. job.ignore = true;
  320. return;
  321. }
  322. if (job === this[CURRENT] && !job.piped) {
  323. this[PIPE](job);
  324. }
  325. }
  326. [ENTRYOPT](job) {
  327. return {
  328. onwarn: (code, msg, data) => this.warn(code, msg, data),
  329. noPax: this.noPax,
  330. cwd: this.cwd,
  331. absolute: job.absolute,
  332. preservePaths: this.preservePaths,
  333. maxReadSize: this.maxReadSize,
  334. strict: this.strict,
  335. portable: this.portable,
  336. linkCache: this.linkCache,
  337. statCache: this.statCache,
  338. noMtime: this.noMtime,
  339. mtime: this.mtime,
  340. prefix: this.prefix,
  341. onWriteEntry: this.onWriteEntry,
  342. };
  343. }
  344. [ENTRY](job) {
  345. this[JOBS] += 1;
  346. try {
  347. const e = new this[WRITEENTRYCLASS](job.path, this[ENTRYOPT](job));
  348. return e
  349. .on('end', () => this[JOBDONE](job))
  350. .on('error', er => this.emit('error', er));
  351. }
  352. catch (er) {
  353. this.emit('error', er);
  354. }
  355. }
  356. [ONDRAIN]() {
  357. if (this[CURRENT] && this[CURRENT].entry) {
  358. this[CURRENT].entry.resume();
  359. }
  360. }
  361. // like .pipe() but using super, because our write() is special
  362. [PIPE](job) {
  363. job.piped = true;
  364. if (job.readdir) {
  365. job.readdir.forEach(entry => {
  366. const p = job.path;
  367. const base = p === './' ? '' : p.replace(/\/*$/, '/');
  368. this[ADDFSENTRY](base + entry);
  369. });
  370. }
  371. const source = job.entry;
  372. const zip = this.zip;
  373. /* c8 ignore start */
  374. if (!source)
  375. throw new Error('cannot pipe without source');
  376. /* c8 ignore stop */
  377. if (zip) {
  378. source.on('data', chunk => {
  379. if (!zip.write(chunk)) {
  380. source.pause();
  381. }
  382. });
  383. }
  384. else {
  385. source.on('data', chunk => {
  386. if (!super.write(chunk)) {
  387. source.pause();
  388. }
  389. });
  390. }
  391. }
  392. pause() {
  393. if (this.zip) {
  394. this.zip.pause();
  395. }
  396. return super.pause();
  397. }
  398. warn(code, message, data = {}) {
  399. warnMethod(this, code, message, data);
  400. }
  401. }
  402. export class PackSync extends Pack {
  403. sync = true;
  404. constructor(opt) {
  405. super(opt);
  406. this[WRITEENTRYCLASS] = WriteEntrySync;
  407. }
  408. // pause/resume are no-ops in sync streams.
  409. pause() { }
  410. resume() { }
  411. [STAT](job) {
  412. const stat = this.follow ? 'statSync' : 'lstatSync';
  413. this[ONSTAT](job, fs[stat](job.absolute));
  414. }
  415. [READDIR](job) {
  416. this[ONREADDIR](job, fs.readdirSync(job.absolute));
  417. }
  418. // gotta get it all in this tick
  419. [PIPE](job) {
  420. const source = job.entry;
  421. const zip = this.zip;
  422. if (job.readdir) {
  423. job.readdir.forEach(entry => {
  424. const p = job.path;
  425. const base = p === './' ? '' : p.replace(/\/*$/, '/');
  426. this[ADDFSENTRY](base + entry);
  427. });
  428. }
  429. /* c8 ignore start */
  430. if (!source)
  431. throw new Error('Cannot pipe without source');
  432. /* c8 ignore stop */
  433. if (zip) {
  434. source.on('data', chunk => {
  435. zip.write(chunk);
  436. });
  437. }
  438. else {
  439. source.on('data', chunk => {
  440. super[WRITE](chunk);
  441. });
  442. }
  443. }
  444. }
  445. //# sourceMappingURL=pack.js.map