利用 Whisper + DeepSeek + ChatTTS 构建语音对话机器人

pip listPackage Version----------------------------- --------------absl-py 2.0.0accelerate0.25.0aiofiles23.2.1aiohttp 3.8.6aiosignal 1.3.1altair5.1.2annotated-types 0.6.0antlr4-python3-runtime4.9.3anyio 4.0.0argon2-cffi 23.1.0argon2-cffi-bindings21.2.0arrow 1.3.0asttokens 2.4.1astunparse1.6.3async-lru 2.0.4async-timeout 4.0.3attrs 23.1.0audioread 3.0.1Babel 2.15.0backcall0.2.0backports.zoneinfo0.2.1beautifulsoup44.12.3bitarray2.8.2bitsandbytes0.41.1bleach6.1.0blinker 1.6.3cachetools5.3.1cdifflib1.2.6certifi 2023.7.22cffi1.16.0charset-normalizer2.1.1click 8.1.7colorama0.4.6comm0.2.2contourpy 1.1.1cpm-kernels 1.0.11cycler0.12.1Cython3.0.3debugpy 1.8.1decorator 5.1.1defusedxml0.7.1distro1.9.0dlib19.24.2edge-tts6.1.8editdistance0.8.1einops0.8.0einx0.2.2encodec 0.1.1exceptiongroup1.1.3executing 2.0.1face-alignment1.4.1fairseq 0.12.2faiss-cpu 1.7.4fastapi 0.108.0fastjsonschema2.19.1ffmpeg1.4ffmpeg-python 0.2.0ffmpy 0.3.1filelock3.12.4Flask 2.1.2Flask-Cors3.0.10flatbuffers 23.5.26fonttools 4.43.1fqdn1.5.1frozendict2.4.4frozenlist1.4.0fsspec2023.9.2future0.18.3gast0.4.0gitdb 4.0.10GitPython 3.1.37google-auth 2.23.3google-auth-oauthlib1.0.0google-pasta0.2.0gradio4.32.2gradio_client 0.17.0grpcio1.59.0h11 0.14.0h5py3.10.0httpcore0.18.0httpx 0.25.0huggingface-hub 0.23.2hydra-core1.0.7idna3.4imageio 2.31.5importlib-metadata6.8.0importlib-resources 6.1.0inflect 7.2.1ipykernel 6.29.4ipython 8.12.3ipywidgets8.1.3isoduration 20.11.0itsdangerous2.1.2jedi0.19.1Jinja23.1.2joblib1.3.2json5 0.9.25jsonpointer 2.4jsonschema4.19.1jsonschema-specifications 2023.7.1jupyter 1.0.0jupyter_client8.6.2jupyter-console 6.6.3jupyter_core5.7.2jupyter-events0.10.0jupyter-lsp 2.2.5jupyter_server2.14.1jupyter_server_terminals0.5.3jupyterlab4.2.1jupyterlab_pygments 0.3.0jupyterlab_server 2.27.2jupyterlab_widgets3.0.11keras 2.13.1kiwisolver1.4.5langdetect1.0.9latex2mathml3.77.0lazy_loader 0.3libclang16.0.6librosa 0.9.1llvmlite0.41.0loguru0.7.2lxml4.9.3Markdown3.5markdown-it-py3.0.0MarkupSafe2.1.3matplotlib3.7.3matplotlib-inline 0.1.7mdtex2html1.2.0mdurl 0.1.2mistune 3.0.2more-itertools10.1.0mpmath1.3.0multidict 6.0.4nbclient0.10.0nbconvert 7.16.4nbformat5.10.4nemo_text_processing1.0.2nest-asyncio1.6.0networkx3.1notebook7.2.0notebook_shim 0.2.4numba 0.58.0numpy 1.22.4oauthlib3.2.2omegaconf 2.3.0onnx1.14.1onnxoptimizer 0.3.13onnxsim 0.4.33openai1.6.1openai-whisper20230918opencv-python 4.8.1.78opt-einsum3.3.0orjson3.9.9overrides 7.7.0packaging 23.2pandas2.0.3pandocfilters 1.5.1parso 0.8.4peft0.7.1pickleshare 0.7.5Pillow10.0.1pip 24.0pkgutil_resolve_name1.3.10platformdirs3.11.0playsound 1.3.0pooch 1.7.0portalocker 2.8.2praat-parselmouth 0.4.3prometheus_client 0.20.0prompt_toolkit3.0.45protobuf4.25.1psutil5.9.5pure-eval 0.2.2pyarrow 13.0.0pyasn10.5.0pyasn1-modules0.3.0PyAudio 0.2.12pycparser 2.21pydantic2.5.3pydantic_core 2.14.6pydeck0.8.1b0pydub 0.25.1Pygments2.16.1pynini2.1.5pynvml11.5.0PyOpenGL3.1.7pyparsing 3.1.1python-dateutil 2.8.2python-json-logger2.0.7python-multipart0.0.9pytz2023.3.post1PyWavelets1.4.1pywin32 306pywinpty2.0.13pyworld 0.3.0PyYAML6.0.1pyzmq 26.0.3qtconsole 5.5.2QtPy2.4.1referencing 0.30.2regex 2023.10.3requests2.32.3requests-oauthlib 1.3.1resampy 0.4.2rfc3339-validator 0.1.4rfc3986-validator 0.1.1rich13.6.0rpds-py 0.10.4rsa 4.9ruff0.4.7sacrebleu 2.3.1sacremoses0.1.1safetensors 0.4.3scikit-image0.18.1scikit-learn1.3.1scikit-maad 1.3.12scipy 1.7.3semantic-version2.10.0Send2Trash1.8.3sentencepiece 0.1.99setuptools69.5.1shellingham 1.5.4six 1.16.0smmap 5.0.1sniffio 1.3.0sounddevice 0.4.5SoundFile 0.10.3.post1soupsieve 2.5sse-starlette 1.8.2stack-data0.6.3starlette 0.32.0.post1streamlit 1.29.0sympy 1.12tabulate0.9.0tenacity8.2.3tensorboard 2.13.0tensorboard-data-server 0.7.1tensorboardX2.6.2.2tensorflow2.13.0tensorflow-estimator2.13.0tensorflow-intel2.13.0tensorflow-io-gcs-filesystem0.31.0termcolor 2.3.0terminado 0.18.1threadpoolctl 3.2.0tifffile2023.7.10tiktoken0.3.3timm0.9.12tinycss21.3.0tokenizers0.19.1toml0.10.2tomli 2.0.1tomlkit 0.12.0toolz 0.12.0torch 2.1.0+cu121torchaudio2.1.0+cu121torchcrepe0.0.22torchvision 0.16.0+cu121tornado 6.3.3tqdm4.63.0traitlets 5.14.3transformers4.41.2transformers-stream-generator 0.0.4trimesh 4.0.0typeguard 4.3.0typer 0.12.3types-python-dateutil 2.9.0.20240316typing_extensions 4.12.0tzdata2023.3tzlocal 5.1uri-template1.3.0urllib3 2.2.1uvicorn 0.25.0validators0.22.0vector_quantize_pytorch 1.14.8vocos 0.1.0watchdog3.0.0wcwidth 0.2.13webcolors 1.13webencodings0.5.1websocket-client1.8.0websockets11.0.3Werkzeug3.0.0WeTextProcessing0.1.12wget3.2wheel 0.43.0widgetsnbextension4.0.11win32-setctime1.1.0wrapt 1.15.0yarl1.9.2zipp3.17.0

import gradio as grfrom transformers import pipelineimport numpy as np
from ChatTTS.experimental.llm import llm_apiimport ChatTTS
chat = ChatTTS.Chat()chat.load_models(compile=False) # 设置为True以获得更快速度

API_KEY = 'sk-xxxxxxxx' # 需要自行到 https://platform.deepseek.com/api_keys 申请client = llm_api(api_key=API_KEY,base_url="https://api.deepseek.com",model="deepseek-chat")

transcriber = pipeline("automatic-speech-recognition", model="openai/whisper-base")
def transcribe(audio):sr, y = audioy = y.astype(np.float32)y /= np.max(np.abs(y))user_question = transcriber({"sampling_rate": sr, "raw": y})["text"]text = client.call(user_question, prompt_version = 'deepseek')wav = chat.infer(text, use_decoder=True)audio_data = np.array(wav[0]).flatten()sample_rate = 24000
return (sample_rate, audio_data)

demo = gr.Interface(transcribe,gr.Audio(sources=["microphone"]),"audio",)
demo.launch()