Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import threading
2import time
4import ipywidgets as widgets
7def create_gpu_status_widget():
8 utilization_bar = widgets.FloatProgress(
9 value=0,
10 min=0,
11 max=100,
12 step=0.1,
13 bar_style="info", # 'success', 'info', 'warning', 'danger' or ''
14 orientation="horizontal",
15 layout=widgets.Layout(width="auto"),
16 )
18 memory_bar = widgets.FloatProgress(
19 value=0,
20 min=0,
21 max=100,
22 step=0.1,
23 bar_style="info", # 'success', 'info', 'warning', 'danger' or ''
24 orientation="horizontal",
25 layout=widgets.Layout(width="auto"),
26 )
28 try:
29 monitor_gpu_utilization(utilization_bar, memory_bar)
30 error_message = ""
31 not_available_message = ""
32 except Exception as e:
33 error_message = f"({type(e)} - {e}"
34 not_available_message = " (not available)"
36 gpu_status_widget = widgets.VBox(
37 [
38 widgets.VBox(
39 [
40 widgets.Label(
41 f"GPU core utilization:{not_available_message}",
42 layout=widgets.Layout(margin="0px 0px -5px 0px"),
43 ),
44 utilization_bar,
45 ],
46 layout=widgets.Layout(width="auto", justify_content="center"),
47 ),
48 widgets.VBox(
49 [
50 widgets.Label(
51 f"GPU memory utilization:{not_available_message}",
52 layout=widgets.Layout(margin="0px 0px -5px 0px"),
53 ),
54 memory_bar,
55 ],
56 layout=widgets.Layout(width="auto", justify_content="center"),
57 ),
58 ]
59 )
61 return gpu_status_widget, error_message
64def monitor_gpu_utilization(utilization_bar, memory_bar):
65 try:
66 import pynvml
68 pynvml.nvmlInit()
69 handle = pynvml.nvmlDeviceGetHandleByIndex(0)
70 except Exception:
71 raise
73 def update_gpu_status():
74 while True:
75 # cpu_proc = pynvml.nvmlDeviceGetComputeRunningProcesses(handle)
77 # gpu_procs = pynvml.nvmlDeviceGetGraphicsRunningProcesses(handle)
78 # num_procs = len(gpu_procs)
80 # info = pynvml.nvmlDeviceGetMemoryInfo(handle)
81 # print("Total memory:", info.total)
82 # print("Free memory:", info.free)
83 # print("Used memory:", info.used)
85 res = pynvml.nvmlDeviceGetUtilizationRates(handle)
87 utilization_bar.value = res.gpu
88 if res.gpu > 70 and utilization_bar.bar_style == "info":
89 utilization_bar.bar_style = "warning"
90 elif res.gpu <= 70 and utilization_bar.bar_style == "warning":
91 utilization_bar.bar_style = "info"
93 memory_bar.value = res.memory
94 if res.memory > 70 and memory_bar.bar_style == "info":
95 memory_bar.bar_style = "warning"
96 elif res.memory <= 70 and memory_bar.bar_style == "warning":
97 memory_bar.bar_style = "info"
99 time.sleep(1.0)
101 # update_gpu_status()
102 thread = threading.Thread(target=update_gpu_status, daemon=True)
103 thread.start()