Feat: hf llm plugin

livekit-plugins/livekit-plugins-hf/README.md

@@ -0,0 +1,31 @@
+# LiveKit Plugins HuggingFace
+
+Agent Framework plugin for local HuggingFace model inference with optional TurboQuant-GPU KV cache compression.
+
+## Installation
+
+```bash
+pip install livekit-plugins-hf
+```
+
+With TurboQuant-GPU support:
+
+```bash
+pip install livekit-plugins-hf[turboquant]
+```
+
+## Usage
+
+```python
+from livekit.plugins.hf import LLM
+
+# Basic local inference
+llm = LLM(model="meta-llama/Llama-3.1-8B-Instruct")
+
+# With TurboQuant-GPU KV cache compression
+llm = LLM(
+    model="meta-llama/Llama-3.1-8B-Instruct",
+    turboquant=True,
+    turboquant_bits=3,
+)
+```

livekit-plugins/livekit-plugins-hf/livekit/plugins/hf/__init__.py

@@ -0,0 +1,45 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""HuggingFace plugin for LiveKit Agents — local GPU inference with optional TurboQuant-GPU."""
+
+from .llm import LLM, LLMStream
+from .log import logger
+from .version import __version__
+
+__all__ = [
+    "LLM",
+    "LLMStream",
+    "logger",
+    "__version__",
+]
+
+from livekit.agents import Plugin
+
+
+class HFPlugin(Plugin):
+    def __init__(self) -> None:
+        super().__init__(__name__, __version__, __package__, logger)
+
+
+Plugin.register_plugin(HFPlugin())
+
+# Cleanup docs of unexported modules
+_module = dir()
+NOT_IN_ALL = [m for m in _module if m not in __all__]
+
+__pdoc__ = {}
+
+for n in NOT_IN_ALL:
+    __pdoc__[n] = False