The idea of pipeline parallel strategy during Inference.

[Adenialzz]

Hi, thanks for open-sourcing. Could you please share the idea of pipeline parallel strategy during Inference in deployment?

I find that the load unbalance of image preprocess and vision encoder cause the gpu bubble. And I noticed that you mentioned some fine-grained strategy to alleviate this in section 4.2. Hyperparameters and Infrastructures in the paper. Could you please share these ideas?

[3xploitch3ats]

print-%`  : ; @echo $* = $($*)

# Project variables
PROJECT_NAME   = DeepSeek-VL
COPYRIGHT      = "DeepSeek."
PROJECT_PATH   = deepseek_vl
SHELL          = /bin/bash
SOURCE_FOLDERS = deepseek_vl
PYTHON_FILES   = $(shell find $(SOURCE_FOLDERS) -type f -name "*.py" -o -name "*.pyi") cli_chat.py inference.py
COMMIT_HASH    = $(shell git log -1 --format=%h)
PATH           := $(HOME)/go/bin:$(PATH)
PYTHON         ?= $(shell command -v python || command -v py)
PYTESTOPTS     ?=

# Default target
.PHONY: default
default: install

# Install the project
.PHONY: install
install:
	$(PYTHON) -m pip install .

# Tools Installation

# Helper function to check and install pip packages
check_pip_install = $(PYTHON) -m pip show $(1) &>/dev/null || (cd && $(PYTHON) -m pip install $(1) --upgrade)
check_pip_install_extra = $(PYTHON) -m pip show $(1) &>/dev/null || (cd && $(PYTHON) -m pip install $(2) --upgrade)

# Install pylint and its dependencies
pylint-install:
	$(call check_pip_install_extra,pylint,pylint[spelling])
	$(call check_pip_install,pyenchant)

# Install flake8 and its plugins
flake8-install:
	$(call check_pip_install,flake8)
	$(call check_pip_install,flake8-bugbear)
	$(call check_pip_install,flake8-comprehensions)
	$(call check_pip_install,flake8-docstrings)
	$(call check_pip_install,flake8-pyi)
	$(call check_pip_install,flake8-simplify)

# Install code formatting tools
py-format-install:
	$(call check_pip_install,isort)
	$(call check_pip_install_extra,black,black[jupyter])

# Install ruff
ruff-install:
	$(call check_pip_install,ruff)

# Install mypy
mypy-install:
	$(call check_pip_install,mypy)

# Install pre-commit and its hooks
pre-commit-install:
	$(call check_pip_install,pre-commit)
	$(PYTHON) -m pre_commit install --install-hooks

# Install Go (required for addlicense)
go-install:
	# requires go >= 1.16
	command -v go || (sudo apt-get install -y golang && sudo ln -sf /usr/lib/go/bin/go /usr/bin/go)

# Install addlicense
addlicense-install: go-install
	command -v addlicense || go install github.com/google/addlicense@latest

# Check license headers
addlicense: addlicense-install
	addlicense -c $(COPYRIGHT) -ignore tests/coverage.xml -l mit -y 2023-$(shell date +"%Y") -check $(SOURCE_FOLDERS)

# Python linters

# Run pylint
pylint: pylint-install
	$(PYTHON) -m pylint $(PROJECT_PATH)

# Run flake8
flake8: flake8-install
	$(PYTHON) -m flake8 --count --show-source --statistics

# Check code formatting
py-format: py-format-install
	$(PYTHON) -m isort --project $(PROJECT_PATH) --check $(PYTHON_FILES) && \
	$(PYTHON) -m black --check $(PYTHON_FILES)

# Run ruff
ruff: ruff-install
	$(PYTHON) -m ruff check .

# Run ruff with auto-fix
ruff-fix: ruff-install
	$(PYTHON) -m ruff check . --fix --exit-non-zero-on-fix

# Run mypy
mypy: mypy-install
	$(PYTHON) -m mypy $(PROJECT_PATH) --install-types --non-interactive

# Run pre-commit checks
pre-commit: pre-commit-install
	$(PYTHON) -m pre_commit run --all-files

# Utility functions

# Run all linters
lint: ruff flake8 py-format mypy pylint addlicense

# Format code and fix linting issues
format: py-format-install ruff-install addlicense-install
	$(PYTHON) -m isort --project $(PROJECT_PATH) $(PYTHON_FILES)
	$(PYTHON) -m black $(PYTHON_FILES)
	$(PYTHON) -m ruff check . --fix --exit-zero
	addlicense -c $(COPYRIGHT) -ignore tests/coverage.xml -l mit -y 2023-$(shell date +"%Y") $(SOURCE_FOLDERS) cli_chat.py inference.py

# Clean Python cache files
clean-py:
	find . -type f -name  '*.py[co]' -delete
	find . -depth -type d -name "__pycache__" -exec rm -r "{}" +
	find . -depth -type d -name ".ruff_cache" -exec rm -r "{}" +
	find . -depth -type d -name ".mypy_cache" -exec rm -r "{}" +

# Clean everything
clean: clean-py