1. kerastest_migcompat.py<\/strong>: GPU \u3092\u4f7f\u3046 python \u30b3\u30fc\u30c9<\/p>\n\r\nimport os\r\n# Set GPU environment before importing TensorFlow\r\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\r\nos.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'\r\n\r\n# TensorFlow and keras\r\nimport tensorflow as tf\r\nimport keras\r\nimport numpy as np\r\nimport matplotlib.pyplot as plt\r\nfrom tensorflow.python.client import device_lib\r\nprint(tf.__version__)\r\n\r\n# GPU available?\r\nprint(device_lib.list_local_devices())\r\n\r\n# load the dataset \r\nfashion_mnist = keras.datasets.fashion_mnist\r\n(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()\r\n\r\nclass_names = ['T-shirt\/top', 'Trouser', 'Pullover', 'Dress', 'Coat', \r\n 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']\r\n\r\ntrain_images = train_images \/ 255.0\r\ntest_images = test_images \/ 255.0\r\n\r\n# define and train the model\r\nmodel = keras.Sequential([\r\n keras.layers.Flatten(input_shape=(28, 28)),\r\n keras.layers.Dense(8192, activation=tf.nn.relu),\r\n keras.layers.Dense(4096, activation=tf.nn.softmax)\r\n])\r\n\r\nmodel.compile(optimizer='adam',\r\n loss='sparse_categorical_crossentropy',\r\n metrics=['accuracy'])\r\n\r\nmodel.fit(train_images, train_labels, epochs=5)\r\n<\/pre>\n
\n2. Dockerfile<\/strong>: Nvidia \u30c4\u30fc\u30eb\u3068\u304b CUDA \u3068\u304b\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3057\u305f\u30a4\u30e1\u30fc\u30b8\u3092\u4f5c\u308b<\/p>\n\r\n# Use NVIDIA CUDA 11.8 base image with Python 3.11\r\nFROM nvidia\/cuda:11.8.0-cudnn8-runtime-ubuntu22.04\r\n\r\n# Set environment variables\r\nENV DEBIAN_FRONTEND=noninteractive\r\nENV PYTHONUNBUFFERED=1\r\nENV TF_CPP_MIN_LOG_LEVEL=2\r\n\r\n# Install Python 3.11 and system dependencies\r\nRUN apt-get update && apt-get install -y \\\r\n software-properties-common \\\r\n wget \\\r\n curl \\\r\n git \\\r\n && add-apt-repository ppa:deadsnakes\/ppa \\\r\n && apt-get update && apt-get install -y \\\r\n python3.11 \\\r\n python3.11-dev \\\r\n python3.11-distutils \\\r\n python3-pip \\\r\n libgl1-mesa-glx \\\r\n libglib2.0-0 \\\r\n libsm6 \\\r\n libxext6 \\\r\n libxrender-dev \\\r\n libgomp1 \\\r\n && apt-get clean \\\r\n && rm -rf \/var\/lib\/apt\/lists\/*\r\n\r\n# Set Python 3.11 as default\r\nRUN update-alternatives --install \/usr\/bin\/python3 python3 \/usr\/bin\/python3.11 1 \\\r\n && update-alternatives --install \/usr\/bin\/python python \/usr\/bin\/python3.11 1\r\n\r\n# Upgrade pip\r\nRUN python3.11 -m pip install --upgrade pip setuptools wheel\r\n\r\n# Create working directory\r\nWORKDIR \/app\r\n\r\n# Copy requirements file first (for better Docker layer caching)\r\nCOPY requirements.txt \/app\/\r\n\r\n# Install Python packages from requirements.txt\r\nRUN pip install --no-cache-dir -r requirements.txt\r\n\r\n# Copy the Python script and entrypoint\r\nCOPY kerastest_migcompat.py \/app\/\r\nCOPY entrypoint.sh \/app\/\r\n\r\n# Make entrypoint script executable\r\nRUN chmod +x \/app\/entrypoint.sh\r\n\r\n# Create a non-root user\r\nRUN useradd -m -u 1000 mluser && \\\r\n chown -R mluser:mluser \/app\r\n\r\nUSER mluser\r\n\r\n# Use entrypoint to keep container running\r\nENTRYPOINT ["\/app\/entrypoint.sh"]\r\n<\/pre>\n
\n3. entrypoint.sh<\/strong><\/p>\n\r\n#!\/bin\/bash\r\n\r\necho "Container started successfully!"\r\necho "Python version:"\r\npython --version\r\n\r\necho ""\r\necho "TensorFlow GPU check:"\r\npython -c "import tensorflow as tf; print('TensorFlow version:', tf.__version__); print('GPU devices:', tf.config.list_physical_devices('GPU'))"\r\n\r\necho ""\r\necho "NVIDIA GPU information:"\r\nnvidia-smi -L\r\n\r\necho ""\r\necho "Container is ready for interactive use."\r\necho ""\r\necho "Keeping container alive..."\r\n\r\n# Keep the container running\r\ntail -f \/dev\/null\r\n<\/pre>\n <\/p>\n
4. requirements.txt<\/strong><\/p>\n\r\ntensorflow[and-cuda]\r\nkeras\r\nnumpy\r\nmatplotlib\r\nscipy\r\n<\/pre>\n <\/p>\n
5. keras-mig-deployment.yaml<\/strong>: Pod \u3092\u30c7\u30d7\u30ed\u30a4\u3059\u308b\u305f\u3081\u306e YAML<\/p>\n\r\napiVersion: v1\r\nkind: Pod\r\nmetadata:\r\n name: keras-mig-test\r\n labels:\r\n app: keras-mig-test\r\nspec:\r\n restartPolicy: Never\r\n containers:\r\n - name: keras-container\r\n image: orenoacr.azurecr.io\/keras-mig-test:latest #Replace with your registry\r\n imagePullPolicy: Always\r\n resources:\r\n limits:\r\n "nvidia.com\/mig-1g.10gb": 1\r\n volumeMounts:\r\n - name: dshm\r\n mountPath: \/dev\/shm\r\n volumes:\r\n - name: dshm\r\n emptyDir:\r\n medium: Memory\r\n sizeLimit: 2Gi\r\n nodeSelector:\r\n node.kubernetes.io\/instance-type: standard_nc24ads_a100_v4\r\n tolerations:\r\n - key: nvidia.com\/gpu\r\n operator: Exists\r\n effect: NoSchedule\r\n<\/pre>\n <\/p>\n
\u3067\u3001\u30d3\u30eb\u30c9\u3057\u3066\u30c7\u30d7\u30ed\u30a4\u3059\u308b\u3002
\n\u307e\u305a\u306f ACR \u306b\u30ed\u30b0\u30a4\u30f3<\/p>\n
\r\naz acr login --name orenoacr\r\n<\/pre>\n\u7d9a\u3044\u3066 Docker build<\/p>\n
\r\n# Build Docker image\r\ndocker build -t keras-mig-test:latest .\r\n\r\n# Tag for your registry\r\ndocker tag keras-mig-test:latest orenoacr.azurecr.io\/keras-mig-test:latest\r\n\r\n# Push to registry\r\ndocker push orenoacr.azurecr.io\/keras-mig-test:latest\r\n\r\n# Deploy to Kubernetes\r\nkubectl apply -f keras-mig-deployment.yaml\r\n<\/pre>\nPod \u306e\u30ed\u30b0\u78ba\u8a8d<\/p>\n
\r\nkubectl logs keras-mig-test\r\n<\/pre>\n\u305d\u3057\u3066 Pod \u5185\u306e Python\u3000\u30b3\u30fc\u30c9\u3092\u5b9f\u884c\u3057\u3066\u307f\u308b\u3002<\/p>\n
\r\nkubectl exec -it keras-mig-test -- python \/app\/kerastest_migcompat.py\r\n<\/pre>\nGPU \u306e MIG \u30a4\u30f3\u30b9\u30bf\u30f3\u30b9\u3092\u691c\u77e5\u3057\u3066 GPU \u3067\u52d5\u3044\u305f\u3002<\/p>\n
\u51fa\u529b\u7d50\u679c\u306f\u3053\u3046\u306a\u308b\u3002<\/p>\n
\r\nCreated device \/device:GPU:0 with 8098 MB memory: -> device: 0, name: NVIDIA A100 80GB PCIe MIG 1g.10gb, pci bus id: 0001:00:00.0, compute capability: 8.0\r\n[name: "\/device:CPU:0"\r\ndevice_type: "CPU"\r\nmemory_limit: 268435456\r\nlocality {\r\n}\r\nincarnation: 8579157921897532563\r\nxla_global_id: -1\r\n, name: "\/device:GPU:0"\r\ndevice_type: "GPU"\r\nmemory_limit: 8491368448\r\nlocality {\r\n bus_id: 1\r\n links {\r\n }\r\n}\r\nincarnation: 7748880231676912385\r\nphysical_device_desc: "device: 0, name: NVIDIA A100 80GB PCIe MIG 1g.10gb, pci bus id: 0001:00:00.0, compute capability: 8.0"\r\nxla_global_id: 416903419\r\n]\r\n\/usr\/local\/lib\/python3.11\/dist-packages\/keras\/src\/layers\/reshaping\/flatten.py:37: UserWarning: Do not pass an `input_shape`\/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\r\n super().__init__(**kwargs)\r\nI0000 00:00:1749629166.172803 568 gpu_device.cc:2019] Created device \/job:localhost\/replica:0\/task:0\/device:GPU:0 with 8098 MB memory: -> device: 0, name: NVIDIA A100 80GB PCIe MIG 1g.10gb, pci bus id: 0001:00:00.0, compute capability: 8.0\r\nEpoch 1\/5\r\nWARNING: All log messages before absl::InitializeLog() is called are written to STDERR\r\nI0000 00:00:1749629167.419059 672 service.cc:152] XLA service 0x7fee38009730 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\r\nI0000 00:00:1749629167.419087 672 service.cc:160] StreamExecutor device (0): NVIDIA A100 80GB PCIe MIG 1g.10gb, Compute Capability 8.0\r\nI0000 00:00:1749629167.495002 672 cuda_dnn.cc:529] Loaded cuDNN version 90300\r\nI0000 00:00:1749629170.363931 672 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\r\n1875\/1875 \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 20s 9ms\/step - accuracy: 0.7815 - loss: 0.6723\r\nEpoch 2\/5\r\n1875\/1875 \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 17s 9ms\/step - accuracy: 0.8616 - loss: 0.3737\r\nEpoch 3\/5\r\n1875\/1875 \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 17s 9ms\/step - accuracy: 0.8800 - loss: 0.3215\r\nEpoch 4\/5\r\n1875\/1875 \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 17s 9ms\/step - accuracy: 0.8873 - loss: 0.3013\r\nEpoch 5\/5\r\n1875\/1875 \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 17s 9ms\/step - accuracy: 0.8953 - loss: 0.2805\r\n<\/pre>\nMIG \u3067 GPU \u3092\u52b9\u7387\u3088\u304f\u30d6\u30f3\u56de\u3059\u6642\u4ee3\u304c\u6765\u305f\uff08\u8b0e
\n <\/p>\n","protected":false},"excerpt":{"rendered":"
\u524d\u56de\u304b\u3089\u306e\u7d9a\u304d \u524d\u56de: Azure AKS \u3067 Nvidia MIG \u3092\u4f7f\u3046#1 \u4eca\u5ea6\u306f AKS \u306e pod \u4e0a\u3067 Python \u304b\u3089\u3000MIG \u3092\u4f7f\u3046\u3002 \u4ee5\u4e0b\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u7528\u610f\u3002 1. kerastest_migcompa […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[22,31,32,8,29],"tags":[],"class_list":["post-2270","post","type-post","status-publish","format-standard","hentry","category-azure","category-docker","category-kubernetes","category-linux","category-python"],"_links":{"self":[{"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/posts\/2270","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/comments?post=2270"}],"version-history":[{"count":5,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/posts\/2270\/revisions"}],"predecessor-version":[{"id":2277,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/posts\/2270\/revisions\/2277"}],"wp:attachment":[{"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/media?parent=2270"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/categories?post=2270"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.kwonline.org\/memo2\/wp-json\/wp\/v2\/tags?post=2270"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}