File size: 1,485 Bytes
bfaf968 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
// OnDeviceService: uses transformers.js to run a small causal LM in browser
// Requires the transformers.js script loaded in index.html (cdn).
/**
* On device llm inference service using transformers.js
* TODO Implement this class!
*/
export class OnDeviceService {
constructor({modelName = 'distilgpt2'} = {}) {
this.modelName = modelName;
this._ready = false;
this._model = null;
}
/**
* Load the model into memory to be ready for inference.
* Download the model if not already cached. Cache the model for future use.
* TODO Download models from a model hub like HuggingFace using transformers.js
*
* @param progressCb
* @returns {Promise<void>}
*/
async load(progressCb) {
}
/**
* Returns if the model is loaded and ready for inference
* @returns {boolean}
*/
isReady() {
return this._ready;
}
/**
* Perform inference on the on-device model
* TODO Implement inference
*
* @param prompt - The input prompt string
* @param maxNewTokens - Maximum number of new tokens to generate
* @returns {Promise<string>}
*/
async infer(prompt, {maxNewTokens = 50} = {}) {
return "The Answer is 42!";
}
/**
* Update configuration with new values
*
* @param modelName - The name of the model to use
*/
updateConfig({modelName}) {
if (modelName) this.modelName = modelName;
}
} |