Browse Source

Make sentence transformers optional

jherve 1 year ago
parent
commit
da86eb7a06
3 changed files with 11 additions and 185 deletions
  1. 5 1
      pyproject.toml
  2. 3 92
      requirements-dev.lock
  3. 3 92
      requirements.lock

+ 5 - 1
pyproject.toml

@@ -18,7 +18,6 @@ dependencies = [
     "aiolimiter>=1.1.0",
     "yarl>=1.9.4",
     "loguru>=0.7.2",
-    "sentence-transformers>=2.6.1",
     "hypercorn>=0.16.0",
     "fastapi>=0.110.1",
     "jinja2>=3.1.3",
@@ -30,6 +29,11 @@ dependencies = [
 readme = "README.md"
 requires-python = ">= 3.10"
 
+[project.optional-dependencies]
+embeddings = [
+    "sentence-transformers>=2.7.0",
+]
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"

+ 3 - 92
requirements-dev.lock

@@ -82,17 +82,9 @@ fastapi==0.111.0
     # via fastapi-cli
 fastapi-cli==0.0.3
     # via fastapi
-filelock==3.14.0
-    # via huggingface-hub
-    # via torch
-    # via transformers
-    # via triton
 frozenlist==1.4.1
     # via aiohttp
     # via aiosignal
-fsspec==2024.3.1
-    # via huggingface-hub
-    # via torch
 h11==0.14.0
     # via httpcore
     # via hypercorn
@@ -108,10 +100,6 @@ httptools==0.6.1
     # via uvicorn
 httpx==0.27.0
     # via fastapi
-huggingface-hub==0.23.0
-    # via sentence-transformers
-    # via tokenizers
-    # via transformers
 hypercorn==0.16.0
     # via de-quoi-parle-le-monde
 hyperframe==6.0.1
@@ -127,15 +115,12 @@ itsdangerous==2.2.0
 jinja2==3.1.4
     # via de-quoi-parle-le-monde
     # via fastapi
-    # via torch
 jmespath==1.0.1
     # via boto3
     # via botocore
-joblib==1.4.2
-    # via scikit-learn
 loguru==0.7.2
     # via de-quoi-parle-le-monde
-lxml==5.2.1
+lxml==5.2.2
     # via de-quoi-parle-le-monde
 markdown-it-py==3.0.0
     # via rich
@@ -145,54 +130,13 @@ mdurl==0.1.2
     # via markdown-it-py
 motor==3.4.0
     # via aiohttp-client-cache
-mpmath==1.3.0
-    # via sympy
 multidict==6.0.5
     # via aiohttp
     # via yarl
-networkx==3.1
-    # via torch
-numpy==1.24.4
+numpy==1.26.4
     # via faiss-cpu
-    # via scikit-learn
-    # via scipy
-    # via sentence-transformers
-    # via transformers
-nvidia-cublas-cu12==12.1.3.1
-    # via nvidia-cudnn-cu12
-    # via nvidia-cusolver-cu12
-    # via torch
-nvidia-cuda-cupti-cu12==12.1.105
-    # via torch
-nvidia-cuda-nvrtc-cu12==12.1.105
-    # via torch
-nvidia-cuda-runtime-cu12==12.1.105
-    # via torch
-nvidia-cudnn-cu12==8.9.2.26
-    # via torch
-nvidia-cufft-cu12==11.0.2.54
-    # via torch
-nvidia-curand-cu12==10.3.2.106
-    # via torch
-nvidia-cusolver-cu12==11.4.5.107
-    # via torch
-nvidia-cusparse-cu12==12.1.0.106
-    # via nvidia-cusolver-cu12
-    # via torch
-nvidia-nccl-cu12==2.20.5
-    # via torch
-nvidia-nvjitlink-cu12==12.4.127
-    # via nvidia-cusolver-cu12
-    # via nvidia-cusparse-cu12
-nvidia-nvtx-cu12==12.1.105
-    # via torch
 orjson==3.10.3
     # via fastapi
-packaging==24.0
-    # via huggingface-hub
-    # via transformers
-pillow==10.3.0
-    # via sentence-transformers
 platformdirs==4.2.1
     # via requests-cache
 priority==2.0.0
@@ -214,18 +158,12 @@ python-dotenv==1.0.1
 python-multipart==0.0.9
     # via fastapi
 pyyaml==6.0.1
-    # via huggingface-hub
-    # via transformers
     # via uvicorn
 redis==5.0.4
     # via aiohttp-client-cache
-regex==2024.5.10
-    # via transformers
 requests==2.31.0
     # via de-quoi-parle-le-monde
-    # via huggingface-hub
     # via requests-cache
-    # via transformers
 requests-cache==1.2.0
     # via de-quoi-parle-le-monde
 rich==13.7.1
@@ -234,15 +172,6 @@ ruff==0.4.4
     # via de-quoi-parle-le-monde
 s3transfer==0.10.1
     # via boto3
-safetensors==0.4.3
-    # via transformers
-scikit-learn==1.3.2
-    # via sentence-transformers
-scipy==1.10.1
-    # via scikit-learn
-    # via sentence-transformers
-sentence-transformers==2.7.0
-    # via de-quoi-parle-le-monde
 sentencepiece==0.2.0
     # via de-quoi-parle-le-monde
 shellingham==1.5.4
@@ -257,26 +186,10 @@ soupsieve==2.5
     # via beautifulsoup4
 starlette==0.37.2
     # via fastapi
-sympy==1.12
-    # via torch
 taskgroup==0.0.0a4
     # via hypercorn
-threadpoolctl==3.5.0
-    # via scikit-learn
-tokenizers==0.19.1
-    # via transformers
 tomli==2.0.1
     # via hypercorn
-torch==2.3.0
-    # via sentence-transformers
-tqdm==4.66.4
-    # via huggingface-hub
-    # via sentence-transformers
-    # via transformers
-transformers==4.40.2
-    # via sentence-transformers
-triton==2.3.0
-    # via torch
 typer==0.12.3
     # via fastapi-cli
 typing-extensions==4.11.0
@@ -284,10 +197,8 @@ typing-extensions==4.11.0
     # via anyio
     # via cattrs
     # via fastapi
-    # via huggingface-hub
     # via pydantic
     # via pydantic-core
-    # via torch
     # via typer
     # via uvicorn
 ujson==5.9.0
@@ -295,7 +206,7 @@ ujson==5.9.0
 url-normalize==1.4.3
     # via aiohttp-client-cache
     # via requests-cache
-urllib3==1.26.18
+urllib3==2.2.1
     # via botocore
     # via requests
     # via requests-cache

+ 3 - 92
requirements.lock

@@ -82,17 +82,9 @@ fastapi==0.111.0
     # via fastapi-cli
 fastapi-cli==0.0.3
     # via fastapi
-filelock==3.14.0
-    # via huggingface-hub
-    # via torch
-    # via transformers
-    # via triton
 frozenlist==1.4.1
     # via aiohttp
     # via aiosignal
-fsspec==2024.3.1
-    # via huggingface-hub
-    # via torch
 h11==0.14.0
     # via httpcore
     # via hypercorn
@@ -108,10 +100,6 @@ httptools==0.6.1
     # via uvicorn
 httpx==0.27.0
     # via fastapi
-huggingface-hub==0.23.0
-    # via sentence-transformers
-    # via tokenizers
-    # via transformers
 hypercorn==0.16.0
     # via de-quoi-parle-le-monde
 hyperframe==6.0.1
@@ -127,15 +115,12 @@ itsdangerous==2.2.0
 jinja2==3.1.4
     # via de-quoi-parle-le-monde
     # via fastapi
-    # via torch
 jmespath==1.0.1
     # via boto3
     # via botocore
-joblib==1.4.2
-    # via scikit-learn
 loguru==0.7.2
     # via de-quoi-parle-le-monde
-lxml==5.2.1
+lxml==5.2.2
     # via de-quoi-parle-le-monde
 markdown-it-py==3.0.0
     # via rich
@@ -145,54 +130,13 @@ mdurl==0.1.2
     # via markdown-it-py
 motor==3.4.0
     # via aiohttp-client-cache
-mpmath==1.3.0
-    # via sympy
 multidict==6.0.5
     # via aiohttp
     # via yarl
-networkx==3.1
-    # via torch
-numpy==1.24.4
+numpy==1.26.4
     # via faiss-cpu
-    # via scikit-learn
-    # via scipy
-    # via sentence-transformers
-    # via transformers
-nvidia-cublas-cu12==12.1.3.1
-    # via nvidia-cudnn-cu12
-    # via nvidia-cusolver-cu12
-    # via torch
-nvidia-cuda-cupti-cu12==12.1.105
-    # via torch
-nvidia-cuda-nvrtc-cu12==12.1.105
-    # via torch
-nvidia-cuda-runtime-cu12==12.1.105
-    # via torch
-nvidia-cudnn-cu12==8.9.2.26
-    # via torch
-nvidia-cufft-cu12==11.0.2.54
-    # via torch
-nvidia-curand-cu12==10.3.2.106
-    # via torch
-nvidia-cusolver-cu12==11.4.5.107
-    # via torch
-nvidia-cusparse-cu12==12.1.0.106
-    # via nvidia-cusolver-cu12
-    # via torch
-nvidia-nccl-cu12==2.20.5
-    # via torch
-nvidia-nvjitlink-cu12==12.4.127
-    # via nvidia-cusolver-cu12
-    # via nvidia-cusparse-cu12
-nvidia-nvtx-cu12==12.1.105
-    # via torch
 orjson==3.10.3
     # via fastapi
-packaging==24.0
-    # via huggingface-hub
-    # via transformers
-pillow==10.3.0
-    # via sentence-transformers
 platformdirs==4.2.1
     # via requests-cache
 priority==2.0.0
@@ -214,18 +158,12 @@ python-dotenv==1.0.1
 python-multipart==0.0.9
     # via fastapi
 pyyaml==6.0.1
-    # via huggingface-hub
-    # via transformers
     # via uvicorn
 redis==5.0.4
     # via aiohttp-client-cache
-regex==2024.5.10
-    # via transformers
 requests==2.31.0
     # via de-quoi-parle-le-monde
-    # via huggingface-hub
     # via requests-cache
-    # via transformers
 requests-cache==1.2.0
     # via de-quoi-parle-le-monde
 rich==13.7.1
@@ -234,15 +172,6 @@ ruff==0.4.4
     # via de-quoi-parle-le-monde
 s3transfer==0.10.1
     # via boto3
-safetensors==0.4.3
-    # via transformers
-scikit-learn==1.3.2
-    # via sentence-transformers
-scipy==1.10.1
-    # via scikit-learn
-    # via sentence-transformers
-sentence-transformers==2.7.0
-    # via de-quoi-parle-le-monde
 sentencepiece==0.2.0
     # via de-quoi-parle-le-monde
 shellingham==1.5.4
@@ -257,26 +186,10 @@ soupsieve==2.5
     # via beautifulsoup4
 starlette==0.37.2
     # via fastapi
-sympy==1.12
-    # via torch
 taskgroup==0.0.0a4
     # via hypercorn
-threadpoolctl==3.5.0
-    # via scikit-learn
-tokenizers==0.19.1
-    # via transformers
 tomli==2.0.1
     # via hypercorn
-torch==2.3.0
-    # via sentence-transformers
-tqdm==4.66.4
-    # via huggingface-hub
-    # via sentence-transformers
-    # via transformers
-transformers==4.40.2
-    # via sentence-transformers
-triton==2.3.0
-    # via torch
 typer==0.12.3
     # via fastapi-cli
 typing-extensions==4.11.0
@@ -284,10 +197,8 @@ typing-extensions==4.11.0
     # via anyio
     # via cattrs
     # via fastapi
-    # via huggingface-hub
     # via pydantic
     # via pydantic-core
-    # via torch
     # via typer
     # via uvicorn
 ujson==5.9.0
@@ -295,7 +206,7 @@ ujson==5.9.0
 url-normalize==1.4.3
     # via aiohttp-client-cache
     # via requests-cache
-urllib3==1.26.18
+urllib3==2.2.1
     # via botocore
     # via requests
     # via requests-cache