Initial import with skill sheet working
This commit is contained in:
22
node_modules/file-entry-cache/LICENSE
generated
vendored
Normal file
22
node_modules/file-entry-cache/LICENSE
generated
vendored
Normal file
@ -0,0 +1,22 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) Roy Riojas & Jared Wray
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
115
node_modules/file-entry-cache/README.md
generated
vendored
Normal file
115
node_modules/file-entry-cache/README.md
generated
vendored
Normal file
@ -0,0 +1,115 @@
|
||||
# file-entry-cache
|
||||
> Super simple cache for file metadata, useful for process that work on a given series of files
|
||||
> and that only need to repeat the job on the changed ones since the previous run of the process — Edit
|
||||
|
||||
[](https://npmjs.org/package/file-entry-cache)
|
||||
[](https://github.com/jaredwray/file-entry-cache/actions/workflows/tests.yaml)
|
||||
[](https://codecov.io/github/jaredwray/file-entry-cache)
|
||||
[](https://npmjs.com/package/file-entry-cache)
|
||||
|
||||
|
||||
## install
|
||||
|
||||
```bash
|
||||
npm i --save file-entry-cache
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
The module exposes two functions `create` and `createFromFile`.
|
||||
|
||||
## `create(cacheName, [directory, useCheckSum])`
|
||||
- **cacheName**: the name of the cache to be created
|
||||
- **directory**: Optional the directory to load the cache from
|
||||
- **usecheckSum**: Whether to use md5 checksum to verify if file changed. If false the default will be to use the mtime and size of the file.
|
||||
|
||||
## `createFromFile(pathToCache, [useCheckSum])`
|
||||
- **pathToCache**: the path to the cache file (this combines the cache name and directory)
|
||||
- **useCheckSum**: Whether to use md5 checksum to verify if file changed. If false the default will be to use the mtime and size of the file.
|
||||
|
||||
```js
|
||||
// loads the cache, if one does not exists for the given
|
||||
// Id a new one will be prepared to be created
|
||||
var fileEntryCache = require('file-entry-cache');
|
||||
|
||||
var cache = fileEntryCache.create('testCache');
|
||||
|
||||
var files = expand('../fixtures/*.txt');
|
||||
|
||||
// the first time this method is called, will return all the files
|
||||
var oFiles = cache.getUpdatedFiles(files);
|
||||
|
||||
// this will persist this to disk checking each file stats and
|
||||
// updating the meta attributes `size` and `mtime`.
|
||||
// custom fields could also be added to the meta object and will be persisted
|
||||
// in order to retrieve them later
|
||||
cache.reconcile();
|
||||
|
||||
// use this if you want the non visited file entries to be kept in the cache
|
||||
// for more than one execution
|
||||
//
|
||||
// cache.reconcile( true /* noPrune */)
|
||||
|
||||
// on a second run
|
||||
var cache2 = fileEntryCache.create('testCache');
|
||||
|
||||
// will return now only the files that were modified or none
|
||||
// if no files were modified previous to the execution of this function
|
||||
var oFiles = cache.getUpdatedFiles(files);
|
||||
|
||||
// if you want to prevent a file from being considered non modified
|
||||
// something useful if a file failed some sort of validation
|
||||
// you can then remove the entry from the cache doing
|
||||
cache.removeEntry('path/to/file'); // path to file should be the same path of the file received on `getUpdatedFiles`
|
||||
// that will effectively make the file to appear again as modified until the validation is passed. In that
|
||||
// case you should not remove it from the cache
|
||||
|
||||
// if you need all the files, so you can determine what to do with the changed ones
|
||||
// you can call
|
||||
var oFiles = cache.normalizeEntries(files);
|
||||
|
||||
// oFiles will be an array of objects like the following
|
||||
entry = {
|
||||
key: 'some/name/file', the path to the file
|
||||
changed: true, // if the file was changed since previous run
|
||||
meta: {
|
||||
size: 3242, // the size of the file
|
||||
mtime: 231231231, // the modification time of the file
|
||||
data: {} // some extra field stored for this file (useful to save the result of a transformation on the file
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Motivation for this module
|
||||
|
||||
I needed a super simple and dumb **in-memory cache** with optional disk persistence (write-back cache) in order to make
|
||||
a script that will beautify files with `esformatter` to execute only on the files that were changed since the last run.
|
||||
|
||||
In doing so the process of beautifying files was reduced from several seconds to a small fraction of a second.
|
||||
|
||||
This module uses [flat-cache](https://www.npmjs.com/package/flat-cache) a super simple `key/value` cache storage with
|
||||
optional file persistance.
|
||||
|
||||
The main idea is to read the files when the task begins, apply the transforms required, and if the process succeed,
|
||||
then store the new state of the files. The next time this module request for `getChangedFiles` will return only
|
||||
the files that were modified. Making the process to end faster.
|
||||
|
||||
This module could also be used by processes that modify the files applying a transform, in that case the result of the
|
||||
transform could be stored in the `meta` field, of the entries. Anything added to the meta field will be persisted.
|
||||
Those processes won't need to call `getChangedFiles` they will instead call `normalizeEntries` that will return the
|
||||
entries with a `changed` field that can be used to determine if the file was changed or not. If it was not changed
|
||||
the transformed stored data could be used instead of actually applying the transformation, saving time in case of only
|
||||
a few files changed.
|
||||
|
||||
In the worst case scenario all the files will be processed. In the best case scenario only a few of them will be processed.
|
||||
|
||||
## Important notes
|
||||
- The values set on the meta attribute of the entries should be `stringify-able` ones if possible, flat-cache uses `circular-json` to try to persist circular structures, but this should be considered experimental. The best results are always obtained with non circular values
|
||||
- All the changes to the cache state are done to memory first and only persisted after reconcile.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
|
||||
|
||||
291
node_modules/file-entry-cache/cache.js
generated
vendored
Normal file
291
node_modules/file-entry-cache/cache.js
generated
vendored
Normal file
@ -0,0 +1,291 @@
|
||||
var path = require('path');
|
||||
var crypto = require('crypto');
|
||||
|
||||
module.exports = {
|
||||
createFromFile: function (filePath, useChecksum) {
|
||||
var fname = path.basename(filePath);
|
||||
var dir = path.dirname(filePath);
|
||||
return this.create(fname, dir, useChecksum);
|
||||
},
|
||||
|
||||
create: function (cacheId, _path, useChecksum) {
|
||||
var fs = require('fs');
|
||||
var flatCache = require('flat-cache');
|
||||
var cache = flatCache.load(cacheId, _path);
|
||||
var normalizedEntries = {};
|
||||
|
||||
var removeNotFoundFiles = function removeNotFoundFiles() {
|
||||
const cachedEntries = cache.keys();
|
||||
// remove not found entries
|
||||
cachedEntries.forEach(function remover(fPath) {
|
||||
try {
|
||||
fs.statSync(fPath);
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') {
|
||||
cache.removeKey(fPath);
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
removeNotFoundFiles();
|
||||
|
||||
return {
|
||||
/**
|
||||
* the flat cache storage used to persist the metadata of the `files
|
||||
* @type {Object}
|
||||
*/
|
||||
cache: cache,
|
||||
|
||||
/**
|
||||
* Given a buffer, calculate md5 hash of its content.
|
||||
* @method getHash
|
||||
* @param {Buffer} buffer buffer to calculate hash on
|
||||
* @return {String} content hash digest
|
||||
*/
|
||||
getHash: function (buffer) {
|
||||
return crypto.createHash('md5').update(buffer).digest('hex');
|
||||
},
|
||||
|
||||
/**
|
||||
* Return whether or not a file has changed since last time reconcile was called.
|
||||
* @method hasFileChanged
|
||||
* @param {String} file the filepath to check
|
||||
* @return {Boolean} wheter or not the file has changed
|
||||
*/
|
||||
hasFileChanged: function (file) {
|
||||
return this.getFileDescriptor(file).changed;
|
||||
},
|
||||
|
||||
/**
|
||||
* given an array of file paths it return and object with three arrays:
|
||||
* - changedFiles: Files that changed since previous run
|
||||
* - notChangedFiles: Files that haven't change
|
||||
* - notFoundFiles: Files that were not found, probably deleted
|
||||
*
|
||||
* @param {Array} files the files to analyze and compare to the previous seen files
|
||||
* @return {[type]} [description]
|
||||
*/
|
||||
analyzeFiles: function (files) {
|
||||
var me = this;
|
||||
files = files || [];
|
||||
|
||||
var res = {
|
||||
changedFiles: [],
|
||||
notFoundFiles: [],
|
||||
notChangedFiles: [],
|
||||
};
|
||||
|
||||
me.normalizeEntries(files).forEach(function (entry) {
|
||||
if (entry.changed) {
|
||||
res.changedFiles.push(entry.key);
|
||||
return;
|
||||
}
|
||||
if (entry.notFound) {
|
||||
res.notFoundFiles.push(entry.key);
|
||||
return;
|
||||
}
|
||||
res.notChangedFiles.push(entry.key);
|
||||
});
|
||||
return res;
|
||||
},
|
||||
|
||||
getFileDescriptor: function (file) {
|
||||
var fstat;
|
||||
|
||||
try {
|
||||
fstat = fs.statSync(file);
|
||||
} catch (ex) {
|
||||
this.removeEntry(file);
|
||||
return { key: file, notFound: true, err: ex };
|
||||
}
|
||||
|
||||
if (useChecksum) {
|
||||
return this._getFileDescriptorUsingChecksum(file);
|
||||
}
|
||||
|
||||
return this._getFileDescriptorUsingMtimeAndSize(file, fstat);
|
||||
},
|
||||
|
||||
_getFileDescriptorUsingMtimeAndSize: function (file, fstat) {
|
||||
var meta = cache.getKey(file);
|
||||
var cacheExists = !!meta;
|
||||
|
||||
var cSize = fstat.size;
|
||||
var cTime = fstat.mtime.getTime();
|
||||
|
||||
var isDifferentDate;
|
||||
var isDifferentSize;
|
||||
|
||||
if (!meta) {
|
||||
meta = { size: cSize, mtime: cTime };
|
||||
} else {
|
||||
isDifferentDate = cTime !== meta.mtime;
|
||||
isDifferentSize = cSize !== meta.size;
|
||||
}
|
||||
|
||||
var nEntry = (normalizedEntries[file] = {
|
||||
key: file,
|
||||
changed: !cacheExists || isDifferentDate || isDifferentSize,
|
||||
meta: meta,
|
||||
});
|
||||
|
||||
return nEntry;
|
||||
},
|
||||
|
||||
_getFileDescriptorUsingChecksum: function (file) {
|
||||
var meta = cache.getKey(file);
|
||||
var cacheExists = !!meta;
|
||||
|
||||
var contentBuffer;
|
||||
try {
|
||||
contentBuffer = fs.readFileSync(file);
|
||||
} catch (ex) {
|
||||
contentBuffer = '';
|
||||
}
|
||||
|
||||
var isDifferent = true;
|
||||
var hash = this.getHash(contentBuffer);
|
||||
|
||||
if (!meta) {
|
||||
meta = { hash: hash };
|
||||
} else {
|
||||
isDifferent = hash !== meta.hash;
|
||||
}
|
||||
|
||||
var nEntry = (normalizedEntries[file] = {
|
||||
key: file,
|
||||
changed: !cacheExists || isDifferent,
|
||||
meta: meta,
|
||||
});
|
||||
|
||||
return nEntry;
|
||||
},
|
||||
|
||||
/**
|
||||
* Return the list o the files that changed compared
|
||||
* against the ones stored in the cache
|
||||
*
|
||||
* @method getUpdated
|
||||
* @param files {Array} the array of files to compare against the ones in the cache
|
||||
* @returns {Array}
|
||||
*/
|
||||
getUpdatedFiles: function (files) {
|
||||
var me = this;
|
||||
files = files || [];
|
||||
|
||||
return me
|
||||
.normalizeEntries(files)
|
||||
.filter(function (entry) {
|
||||
return entry.changed;
|
||||
})
|
||||
.map(function (entry) {
|
||||
return entry.key;
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* return the list of files
|
||||
* @method normalizeEntries
|
||||
* @param files
|
||||
* @returns {*}
|
||||
*/
|
||||
normalizeEntries: function (files) {
|
||||
files = files || [];
|
||||
|
||||
var me = this;
|
||||
var nEntries = files.map(function (file) {
|
||||
return me.getFileDescriptor(file);
|
||||
});
|
||||
|
||||
//normalizeEntries = nEntries;
|
||||
return nEntries;
|
||||
},
|
||||
|
||||
/**
|
||||
* Remove an entry from the file-entry-cache. Useful to force the file to still be considered
|
||||
* modified the next time the process is run
|
||||
*
|
||||
* @method removeEntry
|
||||
* @param entryName
|
||||
*/
|
||||
removeEntry: function (entryName) {
|
||||
delete normalizedEntries[entryName];
|
||||
cache.removeKey(entryName);
|
||||
},
|
||||
|
||||
/**
|
||||
* Delete the cache file from the disk
|
||||
* @method deleteCacheFile
|
||||
*/
|
||||
deleteCacheFile: function () {
|
||||
cache.removeCacheFile();
|
||||
},
|
||||
|
||||
/**
|
||||
* remove the cache from the file and clear the memory cache
|
||||
*/
|
||||
destroy: function () {
|
||||
normalizedEntries = {};
|
||||
cache.destroy();
|
||||
},
|
||||
|
||||
_getMetaForFileUsingCheckSum: function (cacheEntry) {
|
||||
var contentBuffer = fs.readFileSync(cacheEntry.key);
|
||||
var hash = this.getHash(contentBuffer);
|
||||
var meta = Object.assign(cacheEntry.meta, { hash: hash });
|
||||
delete meta.size;
|
||||
delete meta.mtime;
|
||||
return meta;
|
||||
},
|
||||
|
||||
_getMetaForFileUsingMtimeAndSize: function (cacheEntry) {
|
||||
var stat = fs.statSync(cacheEntry.key);
|
||||
var meta = Object.assign(cacheEntry.meta, {
|
||||
size: stat.size,
|
||||
mtime: stat.mtime.getTime(),
|
||||
});
|
||||
delete meta.hash;
|
||||
return meta;
|
||||
},
|
||||
|
||||
/**
|
||||
* Sync the files and persist them to the cache
|
||||
* @method reconcile
|
||||
*/
|
||||
reconcile: function (noPrune) {
|
||||
removeNotFoundFiles();
|
||||
|
||||
noPrune = typeof noPrune === 'undefined' ? true : noPrune;
|
||||
|
||||
var entries = normalizedEntries;
|
||||
var keys = Object.keys(entries);
|
||||
|
||||
if (keys.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
var me = this;
|
||||
|
||||
keys.forEach(function (entryName) {
|
||||
var cacheEntry = entries[entryName];
|
||||
|
||||
try {
|
||||
var meta = useChecksum
|
||||
? me._getMetaForFileUsingCheckSum(cacheEntry)
|
||||
: me._getMetaForFileUsingMtimeAndSize(cacheEntry);
|
||||
cache.setKey(entryName, meta);
|
||||
} catch (err) {
|
||||
// if the file does not exists we don't save it
|
||||
// other errors are just thrown
|
||||
if (err.code !== 'ENOENT') {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
cache.save(noPrune);
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
56
node_modules/file-entry-cache/package.json
generated
vendored
Normal file
56
node_modules/file-entry-cache/package.json
generated
vendored
Normal file
@ -0,0 +1,56 @@
|
||||
{
|
||||
"name": "file-entry-cache",
|
||||
"version": "8.0.0",
|
||||
"description": "Super simple cache for file metadata, useful for process that work o a given series of files and that only need to repeat the job on the changed ones since the previous run of the process",
|
||||
"repository": "jaredwray/file-entry-cache",
|
||||
"license": "MIT",
|
||||
"author": {
|
||||
"name": "Jared Wray",
|
||||
"url": "https://jaredwray.com"
|
||||
},
|
||||
"main": "cache.js",
|
||||
"files": [
|
||||
"cache.js"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"eslint": "eslint --cache --cache-location=node_modules/.cache/ 'cache.js' 'test/**/*.js' 'perf.js'",
|
||||
"autofix": "npm run eslint -- --fix",
|
||||
"clean": "rimraf ./node_modules ./package-lock.json ./yarn.lock",
|
||||
"test": "npm run eslint --silent && c8 mocha -R spec test/specs",
|
||||
"test:ci": "npm run eslint --silent && c8 --reporter=lcov mocha -R spec test/specs",
|
||||
"perf": "node perf.js"
|
||||
},
|
||||
"prepush": [
|
||||
"npm run eslint --silent"
|
||||
],
|
||||
"precommit": [
|
||||
"npm run eslint --silent"
|
||||
],
|
||||
"keywords": [
|
||||
"file cache",
|
||||
"task cache files",
|
||||
"file cache",
|
||||
"key par",
|
||||
"key value",
|
||||
"cache"
|
||||
],
|
||||
"devDependencies": {
|
||||
"c8": "^8.0.1",
|
||||
"chai": "^4.3.10",
|
||||
"eslint": "^8.56.0",
|
||||
"eslint-config-prettier": "^9.1.0",
|
||||
"eslint-plugin-mocha": "^10.2.0",
|
||||
"eslint-plugin-prettier": "^5.0.1",
|
||||
"glob-expand": "^0.2.1",
|
||||
"mocha": "^10.2.0",
|
||||
"prettier": "^3.1.1",
|
||||
"rimraf": "^5.0.5",
|
||||
"write": "^2.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"flat-cache": "^4.0.0"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user