diff --git a/docs/API/README.md b/docs/API/README.md index 14963a3b..a837766e 100644 --- a/docs/API/README.md +++ b/docs/API/README.md @@ -8,6 +8,11 @@ llama.rn - [LlamaContext](classes/LlamaContext.md) - [SchemaGrammarConverter](classes/SchemaGrammarConverter.md) +- [SchemaGrammarConverterBuiltinRule](classes/SchemaGrammarConverterBuiltinRule.md) + +### Interfaces + +- [SchemaGrammarConverterPropOrder](interfaces/SchemaGrammarConverterPropOrder.md) ### Type Aliases @@ -15,6 +20,19 @@ llama.rn - [CompletionParams](README.md#completionparams) - [ContextParams](README.md#contextparams) - [EmbeddingParams](README.md#embeddingparams) +- [NativeCompletionParams](README.md#nativecompletionparams) +- [NativeCompletionResult](README.md#nativecompletionresult) +- [NativeCompletionResultTimings](README.md#nativecompletionresulttimings) +- [NativeCompletionTokenProb](README.md#nativecompletiontokenprob) +- [NativeCompletionTokenProbItem](README.md#nativecompletiontokenprobitem) +- [NativeContextParams](README.md#nativecontextparams) +- [NativeEmbeddingParams](README.md#nativeembeddingparams) +- [NativeEmbeddingResult](README.md#nativeembeddingresult) +- [NativeLlamaContext](README.md#nativellamacontext) +- [NativeSessionLoadResult](README.md#nativesessionloadresult) +- [NativeTokenizeResult](README.md#nativetokenizeresult) +- [RNLlamaMessagePart](README.md#rnllamamessagepart) +- [RNLlamaOAICompatibleMessage](README.md#rnllamaoaicompatiblemessage) - [TokenData](README.md#tokendata) ### Functions @@ -45,37 +63,317 @@ llama.rn #### Defined in -[index.ts:63](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L63) +[index.ts:84](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L84) ___ ### CompletionParams -Ƭ **CompletionParams**: `Omit`<`NativeCompletionParams`, ``"emit_partial_completion"`` \| ``"prompt"``\> & { `chatTemplate?`: `string` ; `messages?`: `RNLlamaOAICompatibleMessage`[] ; `prompt?`: `string` } +Ƭ **CompletionParams**: `Omit`<[`NativeCompletionParams`](README.md#nativecompletionparams), ``"emit_partial_completion"`` \| ``"prompt"``\> & { `chatTemplate?`: `string` ; `messages?`: [`RNLlamaOAICompatibleMessage`](README.md#rnllamaoaicompatiblemessage)[] ; `prompt?`: `string` } #### Defined in -[index.ts:54](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L54) +[index.ts:75](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L75) ___ ### ContextParams -Ƭ **ContextParams**: `Omit`<`NativeContextParams`, ``"cache_type_k"`` \| ``"cache_type_v"`` \| ``"pooling_type"``\> & { `cache_type_k?`: ``"f16"`` \| ``"f32"`` \| ``"q8_0"`` \| ``"q4_0"`` \| ``"q4_1"`` \| ``"iq4_nl"`` \| ``"q5_0"`` \| ``"q5_1"`` ; `cache_type_v?`: ``"f16"`` \| ``"f32"`` \| ``"q8_0"`` \| ``"q4_0"`` \| ``"q4_1"`` \| ``"iq4_nl"`` \| ``"q5_0"`` \| ``"q5_1"`` ; `pooling_type?`: ``"none"`` \| ``"mean"`` \| ``"cls"`` \| ``"last"`` \| ``"rank"`` } +Ƭ **ContextParams**: `Omit`<[`NativeContextParams`](README.md#nativecontextparams), ``"cache_type_k"`` \| ``"cache_type_v"`` \| ``"pooling_type"``\> & { `cache_type_k?`: ``"f16"`` \| ``"f32"`` \| ``"q8_0"`` \| ``"q4_0"`` \| ``"q4_1"`` \| ``"iq4_nl"`` \| ``"q5_0"`` \| ``"q5_1"`` ; `cache_type_v?`: ``"f16"`` \| ``"f32"`` \| ``"q8_0"`` \| ``"q4_0"`` \| ``"q4_1"`` \| ``"iq4_nl"`` \| ``"q5_0"`` \| ``"q5_1"`` ; `pooling_type?`: ``"none"`` \| ``"mean"`` \| ``"cls"`` \| ``"last"`` \| ``"rank"`` } #### Defined in -[index.ts:43](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L43) +[index.ts:64](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L64) ___ ### EmbeddingParams -Ƭ **EmbeddingParams**: `NativeEmbeddingParams` +Ƭ **EmbeddingParams**: [`NativeEmbeddingParams`](README.md#nativeembeddingparams) + +#### Defined in + +[index.ts:73](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L73) + +___ + +### NativeCompletionParams + +Ƭ **NativeCompletionParams**: `Object` + +#### Type declaration + +| Name | Type | Description | +| :------ | :------ | :------ | +| `dry_allowed_length?` | `number` | Tokens that extend repetition beyond this receive exponentially increasing penalty: multiplier * base ^ (length of repeating sequence before token - allowed length). Default: `2` | +| `dry_base?` | `number` | Set the DRY repetition penalty base value. Default: `1.75` | +| `dry_multiplier?` | `number` | Set the DRY (Don't Repeat Yourself) repetition penalty multiplier. Default: `0.0`, which is disabled. | +| `dry_penalty_last_n?` | `number` | How many tokens to scan for repetitions. Default: `-1`, where `0` is disabled and `-1` is context size. | +| `dry_sequence_breakers?` | `string`[] | Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted. Default: `['\n', ':', '"', '*']` | +| `emit_partial_completion` | `boolean` | - | +| `grammar?` | `string` | Set grammar for grammar-based sampling. Default: no grammar | +| `ignore_eos?` | `boolean` | Ignore end of stream token and continue generating. Default: `false` | +| `logit_bias?` | `number`[][] | Modify the likelihood of a token appearing in the generated text completion. For example, use `"logit_bias": [[15043,1.0]]` to increase the likelihood of the token 'Hello', or `"logit_bias": [[15043,-1.0]]` to decrease its likelihood. Setting the value to false, `"logit_bias": [[15043,false]]` ensures that the token `Hello` is never produced. The tokens can also be represented as strings, e.g.`[["Hello, World!",-0.5]]` will reduce the likelihood of all the individual tokens that represent the string `Hello, World!`, just like the `presence_penalty` does. Default: `[]` | +| `min_p?` | `number` | The minimum probability for a token to be considered, relative to the probability of the most likely token. Default: `0.05` | +| `mirostat?` | `number` | Enable Mirostat sampling, controlling perplexity during text generation. Default: `0`, where `0` is disabled, `1` is Mirostat, and `2` is Mirostat 2.0. | +| `mirostat_eta?` | `number` | Set the Mirostat learning rate, parameter eta. Default: `0.1` | +| `mirostat_tau?` | `number` | Set the Mirostat target entropy, parameter tau. Default: `5.0` | +| `n_predict?` | `number` | Set the maximum number of tokens to predict when generating text. **Note:** May exceed the set limit slightly if the last token is a partial multibyte character. When 0,no tokens will be generated but the prompt is evaluated into the cache. Default: `-1`, where `-1` is infinity. | +| `n_probs?` | `number` | If greater than 0, the response also contains the probabilities of top N tokens for each generated token given the sampling settings. Note that for temperature < 0 the tokens are sampled greedily but token probabilities are still being calculated via a simple softmax of the logits without considering any other sampler settings. Default: `0` | +| `n_threads?` | `number` | - | +| `penalize_nl?` | `boolean` | Penalize newline tokens when applying the repeat penalty. Default: `false` | +| `penalty_freq?` | `number` | Repeat alpha frequency penalty. Default: `0.0`, which is disabled. | +| `penalty_last_n?` | `number` | Last n tokens to consider for penalizing repetition. Default: `64`, where `0` is disabled and `-1` is ctx-size. | +| `penalty_present?` | `number` | Repeat alpha presence penalty. Default: `0.0`, which is disabled. | +| `penalty_repeat?` | `number` | Control the repetition of token sequences in the generated text. Default: `1.0` | +| `prompt` | `string` | - | +| `seed?` | `number` | Set the random number generator (RNG) seed. Default: `-1`, which is a random seed. | +| `stop?` | `string`[] | Specify a JSON array of stopping strings. These words will not be included in the completion, so make sure to add them to the prompt for the next iteration. Default: `[]` | +| `temperature?` | `number` | Adjust the randomness of the generated text. Default: `0.8` | +| `top_k?` | `number` | Limit the next token selection to the K most probable tokens. Default: `40` | +| `top_p?` | `number` | Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P. Default: `0.95` | +| `typical_p?` | `number` | Enable locally typical sampling with parameter p. Default: `1.0`, which is disabled. | +| `xtc_probability?` | `number` | Set the chance for token removal via XTC sampler. Default: `0.0`, which is disabled. | +| `xtc_threshold?` | `number` | Set a minimum probability threshold for tokens to be removed via XTC sampler. Default: `0.1` (> `0.5` disables XTC) | + +#### Defined in + +[NativeRNLlama.ts:50](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L50) + +___ + +### NativeCompletionResult + +Ƭ **NativeCompletionResult**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `completion_probabilities?` | [`NativeCompletionTokenProb`](README.md#nativecompletiontokenprob)[] | +| `stopped_eos` | `boolean` | +| `stopped_limit` | `number` | +| `stopped_word` | `string` | +| `stopping_word` | `string` | +| `text` | `string` | +| `timings` | [`NativeCompletionResultTimings`](README.md#nativecompletionresulttimings) | +| `tokens_cached` | `number` | +| `tokens_evaluated` | `number` | +| `tokens_predicted` | `number` | +| `truncated` | `boolean` | + +#### Defined in + +[NativeRNLlama.ts:195](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L195) + +___ + +### NativeCompletionResultTimings + +Ƭ **NativeCompletionResultTimings**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `predicted_ms` | `number` | +| `predicted_n` | `number` | +| `predicted_per_second` | `number` | +| `predicted_per_token_ms` | `number` | +| `prompt_ms` | `number` | +| `prompt_n` | `number` | +| `prompt_per_second` | `number` | +| `prompt_per_token_ms` | `number` | + +#### Defined in + +[NativeRNLlama.ts:184](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L184) + +___ + +### NativeCompletionTokenProb + +Ƭ **NativeCompletionTokenProb**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `content` | `string` | +| `probs` | [`NativeCompletionTokenProbItem`](README.md#nativecompletiontokenprobitem)[] | + +#### Defined in + +[NativeRNLlama.ts:179](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L179) + +___ + +### NativeCompletionTokenProbItem + +Ƭ **NativeCompletionTokenProbItem**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `prob` | `number` | +| `tok_str` | `string` | + +#### Defined in + +[NativeRNLlama.ts:174](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L174) + +___ + +### NativeContextParams + +Ƭ **NativeContextParams**: `Object` + +#### Type declaration + +| Name | Type | Description | +| :------ | :------ | :------ | +| `cache_type_k?` | `string` | KV cache data type for the K (Experimental in llama.cpp) | +| `cache_type_v?` | `string` | KV cache data type for the V (Experimental in llama.cpp) | +| `embd_normalize?` | `number` | - | +| `embedding?` | `boolean` | - | +| `flash_attn?` | `boolean` | Enable flash attention, only recommended in GPU device (Experimental in llama.cpp) | +| `is_model_asset?` | `boolean` | - | +| `lora?` | `string` | - | +| `lora_scaled?` | `number` | - | +| `model` | `string` | - | +| `n_batch?` | `number` | - | +| `n_ctx?` | `number` | - | +| `n_gpu_layers?` | `number` | - | +| `n_threads?` | `number` | - | +| `pooling_type?` | `number` | - | +| `rope_freq_base?` | `number` | - | +| `rope_freq_scale?` | `number` | - | +| `use_mlock?` | `boolean` | - | +| `use_mmap?` | `boolean` | - | +| `use_progress_callback?` | `boolean` | - | +| `vocab_only?` | `boolean` | - | + +#### Defined in + +[NativeRNLlama.ts:8](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L8) + +___ + +### NativeEmbeddingParams + +Ƭ **NativeEmbeddingParams**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `embd_normalize?` | `number` | + +#### Defined in + +[NativeRNLlama.ts:4](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L4) + +___ + +### NativeEmbeddingResult + +Ƭ **NativeEmbeddingResult**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `embedding` | `number`[] | + +#### Defined in + +[NativeRNLlama.ts:215](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L215) + +___ + +### NativeLlamaContext + +Ƭ **NativeLlamaContext**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `contextId` | `number` | +| `gpu` | `boolean` | +| `model` | `Object` | +| `reasonNoGPU` | `string` | + +#### Defined in + +[NativeRNLlama.ts:219](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L219) + +___ + +### NativeSessionLoadResult + +Ƭ **NativeSessionLoadResult**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `prompt` | `string` | +| `tokens_loaded` | `number` | + +#### Defined in + +[NativeRNLlama.ts:226](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L226) + +___ + +### NativeTokenizeResult + +Ƭ **NativeTokenizeResult**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `tokens` | `number`[] | + +#### Defined in + +[NativeRNLlama.ts:211](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/NativeRNLlama.ts#L211) + +___ + +### RNLlamaMessagePart + +Ƭ **RNLlamaMessagePart**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `text?` | `string` | + +#### Defined in + +[chat.ts:3](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/chat.ts#L3) + +___ + +### RNLlamaOAICompatibleMessage + +Ƭ **RNLlamaOAICompatibleMessage**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `content?` | `string` \| [`RNLlamaMessagePart`](README.md#rnllamamessagepart)[] \| `any` | +| `role` | `string` | #### Defined in -[index.ts:52](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L52) +[chat.ts:7](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/chat.ts#L7) ___ @@ -87,12 +385,12 @@ ___ | Name | Type | | :------ | :------ | -| `completion_probabilities?` | `NativeCompletionTokenProb`[] | +| `completion_probabilities?` | [`NativeCompletionTokenProb`](README.md#nativecompletiontokenprob)[] | | `token` | `string` | #### Defined in -[index.ts:33](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L33) +[index.ts:54](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L54) ## Functions @@ -107,7 +405,7 @@ ___ | `«destructured»` | `Object` | | › `allowFetch?` | `boolean` | | › `dotall?` | `boolean` | -| › `propOrder?` | `PropOrder` | +| › `propOrder?` | [`SchemaGrammarConverterPropOrder`](interfaces/SchemaGrammarConverterPropOrder.md) | | › `schema` | `any` | #### Returns @@ -116,7 +414,7 @@ ___ #### Defined in -[grammar.ts:824](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L824) +[grammar.ts:826](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L826) ___ @@ -137,7 +435,7 @@ ___ #### Defined in -[index.ts:230](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L230) +[index.ts:251](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L251) ___ @@ -157,7 +455,7 @@ ___ #### Defined in -[index.ts:215](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L215) +[index.ts:236](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L236) ___ @@ -171,7 +469,7 @@ ___ #### Defined in -[index.ts:280](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L280) +[index.ts:301](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L301) ___ @@ -191,4 +489,4 @@ ___ #### Defined in -[index.ts:201](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L201) +[index.ts:222](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L222) diff --git a/docs/API/classes/LlamaContext.md b/docs/API/classes/LlamaContext.md index ae5edb5e..52654755 100644 --- a/docs/API/classes/LlamaContext.md +++ b/docs/API/classes/LlamaContext.md @@ -38,11 +38,11 @@ | Name | Type | | :------ | :------ | -| `«destructured»` | `NativeLlamaContext` | +| `«destructured»` | [`NativeLlamaContext`](../README.md#nativellamacontext) | #### Defined in -[index.ts:84](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L84) +[index.ts:105](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L105) ## Properties @@ -52,7 +52,7 @@ #### Defined in -[index.ts:76](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L76) +[index.ts:97](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L97) ___ @@ -62,7 +62,7 @@ ___ #### Defined in -[index.ts:74](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L74) +[index.ts:95](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L95) ___ @@ -78,7 +78,7 @@ ___ #### Defined in -[index.ts:80](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L80) +[index.ts:101](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L101) ___ @@ -88,7 +88,7 @@ ___ #### Defined in -[index.ts:78](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L78) +[index.ts:99](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L99) ## Methods @@ -111,13 +111,13 @@ ___ #### Defined in -[index.ts:176](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L176) +[index.ts:197](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L197) ___ ### completion -▸ **completion**(`params`, `callback?`): `Promise`<`NativeCompletionResult`\> +▸ **completion**(`params`, `callback?`): `Promise`<[`NativeCompletionResult`](../README.md#nativecompletionresult)\> #### Parameters @@ -128,11 +128,11 @@ ___ #### Returns -`Promise`<`NativeCompletionResult`\> +`Promise`<[`NativeCompletionResult`](../README.md#nativecompletionresult)\> #### Defined in -[index.ts:120](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L120) +[index.ts:141](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L141) ___ @@ -152,28 +152,28 @@ ___ #### Defined in -[index.ts:165](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L165) +[index.ts:186](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L186) ___ ### embedding -▸ **embedding**(`text`, `params?`): `Promise`<`NativeEmbeddingResult`\> +▸ **embedding**(`text`, `params?`): `Promise`<[`NativeEmbeddingResult`](../README.md#nativeembeddingresult)\> #### Parameters | Name | Type | | :------ | :------ | | `text` | `string` | -| `params?` | `NativeEmbeddingParams` | +| `params?` | [`NativeEmbeddingParams`](../README.md#nativeembeddingparams) | #### Returns -`Promise`<`NativeEmbeddingResult`\> +`Promise`<[`NativeEmbeddingResult`](../README.md#nativeembeddingresult)\> #### Defined in -[index.ts:169](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L169) +[index.ts:190](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L190) ___ @@ -185,7 +185,7 @@ ___ | Name | Type | | :------ | :------ | -| `messages` | `RNLlamaOAICompatibleMessage`[] | +| `messages` | [`RNLlamaOAICompatibleMessage`](../README.md#rnllamaoaicompatiblemessage)[] | | `template?` | `string` | #### Returns @@ -194,13 +194,13 @@ ___ #### Defined in -[index.ts:110](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L110) +[index.ts:131](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L131) ___ ### loadSession -▸ **loadSession**(`filepath`): `Promise`<`NativeSessionLoadResult`\> +▸ **loadSession**(`filepath`): `Promise`<[`NativeSessionLoadResult`](../README.md#nativesessionloadresult)\> Load cached prompt & completion state from a file. @@ -212,11 +212,11 @@ Load cached prompt & completion state from a file. #### Returns -`Promise`<`NativeSessionLoadResult`\> +`Promise`<[`NativeSessionLoadResult`](../README.md#nativesessionloadresult)\> #### Defined in -[index.ts:94](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L94) +[index.ts:115](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L115) ___ @@ -230,7 +230,7 @@ ___ #### Defined in -[index.ts:196](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L196) +[index.ts:217](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L217) ___ @@ -254,7 +254,7 @@ Save current cached prompt & completion state to a file. #### Defined in -[index.ts:103](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L103) +[index.ts:124](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L124) ___ @@ -268,13 +268,13 @@ ___ #### Defined in -[index.ts:157](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L157) +[index.ts:178](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L178) ___ ### tokenize -▸ **tokenize**(`text`): `Promise`<`NativeTokenizeResult`\> +▸ **tokenize**(`text`): `Promise`<[`NativeTokenizeResult`](../README.md#nativetokenizeresult)\> #### Parameters @@ -284,8 +284,8 @@ ___ #### Returns -`Promise`<`NativeTokenizeResult`\> +`Promise`<[`NativeTokenizeResult`](../README.md#nativetokenizeresult)\> #### Defined in -[index.ts:161](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/index.ts#L161) +[index.ts:182](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/index.ts#L182) diff --git a/docs/API/classes/SchemaGrammarConverter.md b/docs/API/classes/SchemaGrammarConverter.md index 11c9cb2e..c3aa05fb 100644 --- a/docs/API/classes/SchemaGrammarConverter.md +++ b/docs/API/classes/SchemaGrammarConverter.md @@ -42,11 +42,11 @@ | `options` | `Object` | | `options.allow_fetch?` | `boolean` | | `options.dotall?` | `boolean` | -| `options.prop_order?` | `PropOrder` | +| `options.prop_order?` | [`SchemaGrammarConverterPropOrder`](../interfaces/SchemaGrammarConverterPropOrder.md) | #### Defined in -[grammar.ts:211](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L211) +[grammar.ts:213](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L213) ## Properties @@ -56,7 +56,7 @@ #### Defined in -[grammar.ts:201](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L201) +[grammar.ts:203](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L203) ___ @@ -66,17 +66,17 @@ ___ #### Defined in -[grammar.ts:203](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L203) +[grammar.ts:205](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L205) ___ ### \_propOrder -• `Private` **\_propOrder**: `PropOrder` +• `Private` **\_propOrder**: [`SchemaGrammarConverterPropOrder`](../interfaces/SchemaGrammarConverterPropOrder.md) #### Defined in -[grammar.ts:199](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L199) +[grammar.ts:201](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L201) ___ @@ -90,7 +90,7 @@ ___ #### Defined in -[grammar.ts:207](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L207) +[grammar.ts:209](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L209) ___ @@ -100,7 +100,7 @@ ___ #### Defined in -[grammar.ts:209](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L209) +[grammar.ts:211](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L211) ___ @@ -114,7 +114,7 @@ ___ #### Defined in -[grammar.ts:205](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L205) +[grammar.ts:207](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L207) ## Methods @@ -127,7 +127,7 @@ ___ | Name | Type | | :------ | :------ | | `name` | `string` | -| `rule` | `undefined` \| `BuiltinRule` | +| `rule` | `undefined` \| [`SchemaGrammarConverterBuiltinRule`](SchemaGrammarConverterBuiltinRule.md) | #### Returns @@ -135,7 +135,7 @@ ___ #### Defined in -[grammar.ts:693](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L693) +[grammar.ts:695](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L695) ___ @@ -156,7 +156,7 @@ ___ #### Defined in -[grammar.ts:224](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L224) +[grammar.ts:226](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L226) ___ @@ -179,7 +179,7 @@ ___ #### Defined in -[grammar.ts:710](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L710) +[grammar.ts:712](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L712) ___ @@ -200,7 +200,7 @@ ___ #### Defined in -[grammar.ts:312](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L312) +[grammar.ts:314](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L314) ___ @@ -220,7 +220,7 @@ ___ #### Defined in -[grammar.ts:518](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L518) +[grammar.ts:520](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L520) ___ @@ -241,7 +241,7 @@ ___ #### Defined in -[grammar.ts:323](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L323) +[grammar.ts:325](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L325) ___ @@ -255,7 +255,7 @@ ___ #### Defined in -[grammar.ts:813](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L813) +[grammar.ts:815](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L815) ___ @@ -276,7 +276,7 @@ ___ #### Defined in -[grammar.ts:247](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L247) +[grammar.ts:249](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L249) ___ @@ -297,4 +297,4 @@ ___ #### Defined in -[grammar.ts:529](https://github.com/mybigday/llama.rn/blob/8e54cbb/src/grammar.ts#L529) +[grammar.ts:531](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L531) diff --git a/docs/API/classes/SchemaGrammarConverterBuiltinRule.md b/docs/API/classes/SchemaGrammarConverterBuiltinRule.md new file mode 100644 index 00000000..b6b8afc7 --- /dev/null +++ b/docs/API/classes/SchemaGrammarConverterBuiltinRule.md @@ -0,0 +1,51 @@ +[llama.rn](../README.md) / SchemaGrammarConverterBuiltinRule + +# Class: SchemaGrammarConverterBuiltinRule + +## Table of contents + +### Constructors + +- [constructor](SchemaGrammarConverterBuiltinRule.md#constructor) + +### Properties + +- [content](SchemaGrammarConverterBuiltinRule.md#content) +- [deps](SchemaGrammarConverterBuiltinRule.md#deps) + +## Constructors + +### constructor + +• **new SchemaGrammarConverterBuiltinRule**(`content`, `deps`) + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `content` | `string` | +| `deps` | `string`[] | + +#### Defined in + +[grammar.ts:82](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L82) + +## Properties + +### content + +• **content**: `string` + +#### Defined in + +[grammar.ts:78](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L78) + +___ + +### deps + +• **deps**: `string`[] + +#### Defined in + +[grammar.ts:80](https://github.com/mybigday/llama.rn/blob/0c04b5e/src/grammar.ts#L80) diff --git a/docs/API/interfaces/SchemaGrammarConverterPropOrder.md b/docs/API/interfaces/SchemaGrammarConverterPropOrder.md new file mode 100644 index 00000000..767b48f5 --- /dev/null +++ b/docs/API/interfaces/SchemaGrammarConverterPropOrder.md @@ -0,0 +1,7 @@ +[llama.rn](../README.md) / SchemaGrammarConverterPropOrder + +# Interface: SchemaGrammarConverterPropOrder + +## Indexable + +▪ [key: `string`]: `number` diff --git a/src/NativeRNLlama.ts b/src/NativeRNLlama.ts index 04e06e86..5278f394 100644 --- a/src/NativeRNLlama.ts +++ b/src/NativeRNLlama.ts @@ -49,37 +49,124 @@ export type NativeContextParams = { export type NativeCompletionParams = { prompt: string - grammar?: string - stop?: Array // -> antiprompt - n_threads?: number + /** + * Set grammar for grammar-based sampling. Default: no grammar + */ + grammar?: string + /** + * Specify a JSON array of stopping strings. + * These words will not be included in the completion, so make sure to add them to the prompt for the next iteration. Default: `[]` + */ + stop?: Array + /** + * Set the maximum number of tokens to predict when generating text. + * **Note:** May exceed the set limit slightly if the last token is a partial multibyte character. + * When 0,no tokens will be generated but the prompt is evaluated into the cache. Default: `-1`, where `-1` is infinity. + */ n_predict?: number + /** + * If greater than 0, the response also contains the probabilities of top N tokens for each generated token given the sampling settings. + * Note that for temperature < 0 the tokens are sampled greedily but token probabilities are still being calculated via a simple softmax of the logits without considering any other sampler settings. + * Default: `0` + */ n_probs?: number + /** + * Limit the next token selection to the K most probable tokens. Default: `40` + */ top_k?: number + /** + * Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P. Default: `0.95` + */ top_p?: number + /** + * The minimum probability for a token to be considered, relative to the probability of the most likely token. Default: `0.05` + */ min_p?: number - xtc_threshold?: number + /** + * Set the chance for token removal via XTC sampler. Default: `0.0`, which is disabled. + */ xtc_probability?: number + /** + * Set a minimum probability threshold for tokens to be removed via XTC sampler. Default: `0.1` (> `0.5` disables XTC) + */ + xtc_threshold?: number + /** + * Enable locally typical sampling with parameter p. Default: `1.0`, which is disabled. + */ typical_p?: number - temperature?: number // -> temp + /** + * Adjust the randomness of the generated text. Default: `0.8` + */ + temperature?: number + /** + * Last n tokens to consider for penalizing repetition. Default: `64`, where `0` is disabled and `-1` is ctx-size. + */ penalty_last_n?: number + /** + * Control the repetition of token sequences in the generated text. Default: `1.0` + */ penalty_repeat?: number + /** + * Repeat alpha frequency penalty. Default: `0.0`, which is disabled. + */ penalty_freq?: number + /** + * Repeat alpha presence penalty. Default: `0.0`, which is disabled. + */ penalty_present?: number + /** + * Penalize newline tokens when applying the repeat penalty. Default: `false` + */ + penalize_nl?: boolean + /** + * Enable Mirostat sampling, controlling perplexity during text generation. Default: `0`, where `0` is disabled, `1` is Mirostat, and `2` is Mirostat 2.0. + */ mirostat?: number + /** + * Set the Mirostat target entropy, parameter tau. Default: `5.0` + */ mirostat_tau?: number + /** + * Set the Mirostat learning rate, parameter eta. Default: `0.1` + */ mirostat_eta?: number - penalize_nl?: boolean - seed?: number - + /** + * Set the DRY (Don't Repeat Yourself) repetition penalty multiplier. Default: `0.0`, which is disabled. + */ dry_multiplier?: number + /** + * Set the DRY repetition penalty base value. Default: `1.75` + */ dry_base?: number - dry_allowed_length?: number + /** + * Tokens that extend repetition beyond this receive exponentially increasing penalty: multiplier * base ^ (length of repeating sequence before token - allowed length). Default: `2` + */ + dry_allowed_length?: number + /** + * How many tokens to scan for repetitions. Default: `-1`, where `0` is disabled and `-1` is context size. + */ dry_penalty_last_n?: number + /** + * Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted. Default: `['\n', ':', '"', '*']` + */ dry_sequence_breakers?: Array - + /** + * Ignore end of stream token and continue generating. Default: `false` + */ ignore_eos?: boolean + /** + * Modify the likelihood of a token appearing in the generated text completion. + * For example, use `"logit_bias": [[15043,1.0]]` to increase the likelihood of the token 'Hello', or `"logit_bias": [[15043,-1.0]]` to decrease its likelihood. + * Setting the value to false, `"logit_bias": [[15043,false]]` ensures that the token `Hello` is never produced. The tokens can also be represented as strings, + * e.g.`[["Hello, World!",-0.5]]` will reduce the likelihood of all the individual tokens that represent the string `Hello, World!`, just like the `presence_penalty` does. + * Default: `[]` + */ logit_bias?: Array> + /** + * Set the random number generator (RNG) seed. Default: `-1`, which is a random seed. + */ + seed?: number emit_partial_completion: boolean } diff --git a/src/grammar.ts b/src/grammar.ts index 6a845cf5..d65d30ba 100644 --- a/src/grammar.ts +++ b/src/grammar.ts @@ -74,7 +74,7 @@ function buildRepetition( return result } -class BuiltinRule { +export class SchemaGrammarConverterBuiltinRule { content: string deps: string[] @@ -85,9 +85,11 @@ class BuiltinRule { } } +const BuiltinRule = SchemaGrammarConverterBuiltinRule + const UP_TO_15_DIGITS = buildRepetition('[0-9]', 0, 15) -const PRIMITIVE_RULES: { [key: string]: BuiltinRule } = { +const PRIMITIVE_RULES: { [key: string]: SchemaGrammarConverterBuiltinRule } = { boolean: new BuiltinRule('("true" | "false") space', []), 'decimal-part': new BuiltinRule(`[0-9] ${UP_TO_15_DIGITS}`, []), 'integral-part': new BuiltinRule(`[0-9] | [1-9] ${UP_TO_15_DIGITS}`, []), @@ -126,7 +128,7 @@ const PRIMITIVE_RULES: { [key: string]: BuiltinRule } = { } // TODO: support "uri", "email" string formats -const STRING_FORMAT_RULES: { [key: string]: BuiltinRule } = { +const STRING_FORMAT_RULES: { [key: string]: SchemaGrammarConverterBuiltinRule } = { date: new BuiltinRule( '[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( "0" [1-9] | [1-2] [0-9] | "3" [0-1] )', [], @@ -173,7 +175,7 @@ const formatLiteral = (literal: string): string => { const generateConstantRule = (value: any): string => formatLiteral(JSON.stringify(value)) -interface PropOrder { +export interface SchemaGrammarConverterPropOrder { [key: string]: number } @@ -196,7 +198,7 @@ function* groupBy(iterable: Iterable, keyFn: (x: any) => any) { } export class SchemaGrammarConverter { - private _propOrder: PropOrder + private _propOrder: SchemaGrammarConverterPropOrder private _allowFetch: boolean @@ -209,7 +211,7 @@ export class SchemaGrammarConverter { private _refsBeingResolved: Set constructor(options: { - prop_order?: PropOrder + prop_order?: SchemaGrammarConverterPropOrder allow_fetch?: boolean dotall?: boolean }) { @@ -690,7 +692,7 @@ export class SchemaGrammarConverter { } } - _addPrimitive(name: string, rule: BuiltinRule | undefined) { + _addPrimitive(name: string, rule: SchemaGrammarConverterBuiltinRule | undefined) { if (!rule) { throw new Error(`Rule ${name} not known`) } @@ -828,7 +830,7 @@ export const convertJsonSchemaToGrammar = ({ allowFetch, }: { schema: any - propOrder?: PropOrder + propOrder?: SchemaGrammarConverterPropOrder dotall?: boolean allowFetch?: boolean }): string | Promise => { diff --git a/src/index.ts b/src/index.ts index c77655fc..d85d929c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,11 +11,32 @@ import type { NativeEmbeddingResult, NativeSessionLoadResult, NativeEmbeddingParams, + NativeCompletionTokenProbItem, + NativeCompletionResultTimings, } from './NativeRNLlama' +import type { SchemaGrammarConverterPropOrder, SchemaGrammarConverterBuiltinRule } from './grammar' import { SchemaGrammarConverter, convertJsonSchemaToGrammar } from './grammar' -import type { RNLlamaOAICompatibleMessage } from './chat' +import type { RNLlamaMessagePart, RNLlamaOAICompatibleMessage } from './chat' import { formatChat } from './chat' +export type { + NativeContextParams, + NativeLlamaContext, + NativeCompletionParams, + NativeCompletionTokenProb, + NativeCompletionResult, + NativeTokenizeResult, + NativeEmbeddingResult, + NativeSessionLoadResult, + NativeEmbeddingParams, + NativeCompletionTokenProbItem, + NativeCompletionResultTimings, + RNLlamaMessagePart, + RNLlamaOAICompatibleMessage, + SchemaGrammarConverterPropOrder, + SchemaGrammarConverterBuiltinRule, +} + export { SchemaGrammarConverter, convertJsonSchemaToGrammar } const EVENT_ON_INIT_CONTEXT_PROGRESS = '@RNLlama_onInitContextProgress'