On-Device AI 端侧部署

# === llama.cpp Android 部署 ===

# 1. 下载预编译的 Android 二进制文件或自行编译
git clone https://github.com/ggml-org/llama.cpp
cd llama.cpp
# NDK 交叉编译
mkdir build-android && cd build-android
cmake .. -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake \
  -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-28 \
  -DGGML_OPENCL=ON  # 启用 GPU 加速
make -j$(nproc)

# 2. 下载量化模型（推荐 Q4_K_M，平衡质量和速度）
# Qwen2.5-3B: https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF
# LLaMA-3.1-8B: https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF

# 3. Push 到手机运行
adb push llama-cli /data/local/tmp/
adb push qwen2.5-3b-instruct-q4_k_m.gguf /data/local/tmp/
adb shell "cd /data/local/tmp && ./llama-cli \
  -m qwen2.5-3b-instruct-q4_k_m.gguf \
  -p '请解释PID控制器的工作原理' \
  -n 256 --temp 0.7"

# === MLC-LLM 快速开始 ===

# 1. 安装 MLC-LLM
pip install mlc-llm mlc-ai-nightly

# 2. 下载预编译的 Android APK（最快方式）
# https://llm.mlc.ai/docs/deploy/android.html
# 支持 Vulkan GPU 加速

# 3. 或者用 Python 本地推理测试
from mlc_llm import MLCEngine

engine = MLCEngine("HF://mlc-ai/Qwen2.5-3B-Instruct-q4f16_1-MLC")
response = engine.chat.completions.create(
    messages=[{"role": "user", "content": "解释逆运动学的基本概念"}],
    stream=True
)
for chunk in response:
    print(chunk.choices[0].delta.content, end="")

# 4. 编译自定义模型到 Android
mlc_llm compile ./dist/Qwen2.5-3B-Instruct-q4f16_1-MLC \
  --device android --output ./dist/libs/

// === MediaPipe LLM Inference (Android Kotlin) ===

// build.gradle 添加依赖
// implementation("com.google.mediapipe:tasks-genai:latest.release")

import com.google.mediapipe.tasks.genai.llminference.LlmInference

// 初始化 LLM
val options = LlmInference.LlmInferenceOptions.builder()
    .setModelPath("/data/local/tmp/gemma-2b-it-gpu-int4.bin")
    .setMaxTokens(1024)
    .setTemperature(0.7f)
    .setTopK(40)
    .setResultListener { partialResult, done ->
        // 流式输出
        runOnUiThread { textView.append(partialResult) }
    }
    .build()

val llmInference = LlmInference.createFromOptions(context, options)

// 生成回复
llmInference.generateResponseAsync("请解释SLAM算法的工作原理")

# === Qualcomm AI Hub ===
# 安装
pip install qai-hub

# 1. 浏览可用模型
import qai_hub as hub

# 列出支持的设备
devices = hub.get_devices()
for d in devices:
    print(f"{d.name}: {d.os}, {d.chipset}")
# Samsung Galaxy S24 (Snapdragon 8 Gen 3)
# Samsung Galaxy S25 (Snapdragon 8 Elite)
# Xiaomi 15 (Snapdragon 8 Elite)

# 2. 提交模型到真机上跑 Profile
import torch
from qai_hub_models.models.yolov8_det import Model as YoloV8

model = YoloV8.from_pretrained()
input_shape = (1, 3, 640, 640)
sample_input = torch.randn(input_shape)

# 在真机上 Profile 性能
job = hub.submit_profile_job(
    model=model,
    device=hub.Device("Samsung Galaxy S24"),
    input_shapes=dict(image=input_shape),
)
print(job.get_target_model_perf())

# 3. 下载编译好的模型到设备运行
compiled = hub.submit_compile_job(model, device=hub.Device("Samsung Galaxy S24"))
compiled.download("./yolov8_s24.tflite")

# === ExecuTorch (Meta) - LLaMA on Android ===

# 1. 安装
pip install executorch

# 2. 导出 LLaMA 模型为 ExecuTorch 格式
python -m executorch.examples.models.llama.export_llama \
  --checkpoint path/to/llama3-8b \
  --params path/to/params.json \
  -qmode 8da4w \  # 4-bit 量化
  --output llama3_8b_q4.pte

# 3. Android Demo App
# https://github.com/pytorch/executorch/tree/main/examples/demo-apps/android/LlamaDemo
# 支持 XNNPACK (CPU)、Qualcomm QNN (NPU)、MediaTek (NPU)

# 4. iOS 部署
# 使用 Core ML 后端
python -m executorch.examples.models.llama.export_llama \
  --checkpoint path/to/llama3-8b \
  --coreml  # 启用 Core ML 后端