# カーネルログ（/var/log/kern.log）に残された障害シーケンス
18:31:32  kernel: nvidia 0000:05:00.0: PCIe Bus Error: severity=Uncorrectable (Non-Fatal), type=Transaction Layer
18:31:33  kernel: NVRM: Xid (PCI:0000:05:00): 79, GPU has fallen off the bus.
18:31:33  kernel: NVRM: Xid (PCI:0000:01:00 / 05:00 / 08:00): 154, GPU recovery action changed ... 0x2 (Node Reboot Required)
18:31:33  kernel: NVRM: uvm encountered global fatal error 0x60, requiring os reboot to recover.
18:31:33  Xorg:    Failed to allocate push buffer -> Error recovery failed -> Fatal server error

# 1. パッケージ更新と必須ビルドツールの導入
sudo apt update && sudo apt upgrade -y
sudo apt install -y build-essential dkms git curl wget jq pciutils

2. NVIDIA 公式ドライバ（595.xx / 550+）のインストール
sudo add-apt-repository ppa:graphics-drivers/ppa -y
sudo apt update
sudo apt install -y nvidia-driver-595 nvidia-utils-595

3. 再起動してGPU認識を確認
sudo reboot

再起動後、3台のGPUが認識されているか確認
nvidia-smi --query-gpu=index,name,pci.bus_id,driver_version,memory.total --format=csv

/* /usr/local/src/gpu_dual_unlock.c */
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <dirent.h>
#include <string.h>

#define NVIDIA_VENDOR_ID 0x10de
#define DEVID_RTX4070TIS 0x2705
#define DEVID_RTX3060    0x2503

static void unlock_device(const char* pci_name, uint16_t dev_id) {
    char config_path[256];
    snprintf(config_path, sizeof(config_path), "/sys/bus/pci/devices/%s/config", pci_name);
    
    int fd = open(config_path, O_RDWR);
    if (fd < 0) return;
    
    /* Zagitta & Ada派生: FP32アキュムレーションレートレジスタ解除 */
    uint32_t val = 0;
    lseek(fd, 0x88, SEEK_SET);
    read(fd, &val, sizeof(val));
    val |= (1 << 4); /* Rate throttle clear */
    lseek(fd, 0x88, SEEK_SET);
    write(fd, &val, sizeof(val));
    
    close(fd);
    printf("[gpu-unlock] Unlocked PCIe %s (Device ID: 0x%04x)\n", pci_name, dev_id);
}

int main(void) {
    DIR *dir = opendir("/sys/bus/pci/devices");
    if (!dir) return 1;
    struct dirent *ent;
    while ((ent = readdir(dir)) != NULL) {
        if (ent->d_name[0] == '.') continue;
        char vendor_path[256], dev_path[256];
        snprintf(vendor_path, sizeof(vendor_path), "/sys/bus/pci/devices/%s/vendor", ent->d_name);
        snprintf(dev_path, sizeof(dev_path), "/sys/bus/pci/devices/%s/device", ent->d_name);
        
        FILE *fv = fopen(vendor_path, "r");
        FILE *fd = fopen(dev_path, "r");
        if (fv && fd) {
            uint32_t vendor = 0, device = 0;
            fscanf(fv, "0x%x", &vendor);
            fscanf(fd, "0x%x", &device);
            if (vendor == NVIDIA_VENDOR_ID) {
                unlock_device(ent->d_name, (uint16_t)device);
            }
            fclose(fv); fclose(fd);
        }
    }
    closedir(dir);
    return 0;
}

# コンパイルと配置
sudo gcc -O2 -static /usr/local/src/gpu_dual_unlock.c -o /usr/local/bin/gpu_dual_unlock

initramfs フックスクリプトの作成
sudo tee /etc/initramfs-tools/hooks/gpu_unlock << 'EOF'
#!/bin/sh
PREREQ=""
prereqs() { echo "$PREREQ"; }
case $1 in prereqs) prereqs; exit 0;; esac
. /usr/share/initramfs-tools/hook-functions
copy_exec /usr/local/bin/gpu_dual_unlock /bin
EOF
sudo chmod +x /etc/initramfs-tools/hooks/gpu_unlock

initramfs 起動スクリプトの作成
sudo tee /etc/initramfs-tools/scripts/init-premount/gpu_unlock << 'EOF'
#!/bin/sh
PREREQ=""
prereqs() { echo "$PREREQ"; }
case $1 in prereqs) prereqs; exit 0;; esac
/bin/gpu_dual_unlock
EOF
sudo chmod +x /etc/initramfs-tools/scripts/init-premount/gpu_unlock

initramfs の更新
sudo update-initramfs -u -k all

# 安全検査付き実行シェルスクリプト (run-gemm.sh)
cat << 'EOF' > run-gemm.sh
#!/bin/bash
set -e

DEV_ID=${1:-0}
PCI_BUS=$(nvidia-smi -i $DEV_ID --query-gpu=pci.bus_id --format=csv,noheader | tr '[:upper:]' '[:lower:]')

1. PCIe リンク状態の事前チェック（Gen1 x4等への劣化を拒否）
LINK_SPEED=$(cat /sys/bus/pci/devices/$PCI_BUS/current_link_speed 2>/dev/null || echo "unknown")
echo "[Pre-Check] Device $DEV_ID ($PCI_BUS) Link Speed: $LINK_SPEED"
if [[ "$LINK_SPEED" == "2.5 GT/s"* ]]; then
    echo "ERROR: GPU link degraded to Gen1! Aborting to prevent Xid 79." >&2
    exit 3
fi

2. Xid エラー検知確認
PRE_XID=$(dmesg | grep -c "NVRM: Xid" || true)

3. 段階的GEMM実行 (1024 -> 2048 -> 4096)
./gemm_bench --device $DEV_ID --size 2048 --modes BF16,FP16,TF32

4. 事後検査
POST_XID=$(dmesg | grep -c "NVRM: Xid" || true)
if [ "$POST_XID" -gt "$PRE_XID" ]; then
    echo "WARNING: Xid error logged during benchmark!" >&2
fi
EOF
chmod +x run-gemm.sh

/* strata_config.json */
{
  "server": {
    "host": "0.0.0.0",
    "port": 47832,
    "model": "qwen3.8-flash-next-iq3_xxs",
    "max_context": 32768
  },
  "gpu_roles": {
    "0": "main",    /* RTX 4070 Ti SUPER (16GB) */
    "1": "off",     /* RTX 4060 (安全のため除外) */
    "2": "helper"   /* RTX 3060 (12GB) */
  }
}

# Strata 推論サーバーの起動
./strata-server   --config strata_config.json   --host 0.0.0.0   --port 47832

起動後、Tailscale IP（例: 100.86.91.70）から即座に確認
curl -s http://100.86.91.70:47832/v1/models | jq .

チャット推論テスト
curl -s -X POST http://100.86.91.70:47832/v1/chat/completions   -H "Content-Type: application/json"   -d '{"messages": [{"role": "user", "content": "こんにちは！"}], "max_tokens": 50}' | jq .