Traceback (most recent call last):
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/utils.py", line 49, in tokenizer
    import tiktoken
ModuleNotFoundError: No module named 'tiktoken'

In reply to @pareto-optimal-dev:matrix.org
is it because it wants version 0.3.3?

result/lib/python3.11/site-packages/private_gpt
components  constants.py  di.py  __init__.py  __main__.py  main.py  open_ai  paths.py  __pycache__  server  settings  ui  utils

full trace

❯ python -m private_gpt
Traceback (most recent call last):
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/utils.py", line 49, in tokenizer
    import tiktoken
ModuleNotFoundError: No module named 'tiktoken'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/matthew/git/privateGPT/private_gpt/__main__.py", line 5, in <module>
    from private_gpt.main import app
  File "/home/matthew/git/privateGPT/private_gpt/main.py", line 4, in <module>
    import llama_index
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/__init__.py", line 21, in <module>
    from llama_index.indices.common.struct_store.base import SQLDocumentContextBuilder
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/indices/__init__.py", line 4, in <module>
    from llama_index.indices.document_summary.base import DocumentSummaryIndex
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/indices/document_summary/__init__.py", line 4, in <module>
    from llama_index.indices.document_summary.base import (
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/indices/document_summary/base.py", line 14, in <module>
    from llama_index.indices.base import BaseIndex
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/indices/base.py", line 6, in <module>
    from llama_index.chat_engine.types import BaseChatEngine, ChatMode
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/chat_engine/__init__.py", line 1, in <module>
    from llama_index.chat_engine.condense_question import CondenseQuestionChatEngine
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/chat_engine/condense_question.py", line 6, in <module>
    from llama_index.chat_engine.types import (
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/chat_engine/types.py", line 11, in <module>
    from llama_index.memory import BaseMemory
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/memory/__init__.py", line 1, in <module>
    from llama_index.memory.chat_memory_buffer import ChatMemoryBuffer
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/memory/chat_memory_buffer.py", line 12, in <module>
    class ChatMemoryBuffer(BaseMemory):
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/memory/chat_memory_buffer.py", line 18, in ChatMemoryBuffer
    default_factory=cast(Callable[[], Any], GlobalsHelper().tokenizer),
                                            ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/3g04wving6mlz1vxmnaqxy1wfsyh8g50-python3.11-llama-index-0.8.47/lib/python3.11/site-packages/llama_index/utils.py", line 51, in tokenizer
    raise ImportError(tiktoken_import_err)
ImportError: `tiktoken` package not found, please run `pip install tiktoken`

    def tokenizer(self) -> Callable[[str], List]:
        """Get tokenizer."""
        if self._tokenizer is None:
            tiktoken_import_err = (
                "`tiktoken` package not found, please run `pip install tiktoken`"
            )
            try:
                import tiktoken
            except ImportError:
                raise ImportError(tiktoken_import_err)
            enc = tiktoken.get_encoding("gpt2")
            self._tokenizer = cast(Callable[[str], List], enc.encode)
            self._tokenizer = partial(self._tokenizer, allowed_special="all")
        return self._tokenizer  # type: ignore