Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import threading 

2import time 

3 

4import ipywidgets as widgets 

5 

6 

7def create_gpu_status_widget(): 

8 utilization_bar = widgets.FloatProgress( 

9 value=0, 

10 min=0, 

11 max=100, 

12 step=0.1, 

13 bar_style="info", # 'success', 'info', 'warning', 'danger' or '' 

14 orientation="horizontal", 

15 layout=widgets.Layout(width="auto"), 

16 ) 

17 

18 memory_bar = widgets.FloatProgress( 

19 value=0, 

20 min=0, 

21 max=100, 

22 step=0.1, 

23 bar_style="info", # 'success', 'info', 'warning', 'danger' or '' 

24 orientation="horizontal", 

25 layout=widgets.Layout(width="auto"), 

26 ) 

27 

28 try: 

29 monitor_gpu_utilization(utilization_bar, memory_bar) 

30 error_message = "" 

31 not_available_message = "" 

32 except Exception as e: 

33 error_message = f"({type(e)} - {e}" 

34 not_available_message = " (not available)" 

35 

36 gpu_status_widget = widgets.VBox( 

37 [ 

38 widgets.VBox( 

39 [ 

40 widgets.Label( 

41 f"GPU core utilization:{not_available_message}", 

42 layout=widgets.Layout(margin="0px 0px -5px 0px"), 

43 ), 

44 utilization_bar, 

45 ], 

46 layout=widgets.Layout(width="auto", justify_content="center"), 

47 ), 

48 widgets.VBox( 

49 [ 

50 widgets.Label( 

51 f"GPU memory utilization:{not_available_message}", 

52 layout=widgets.Layout(margin="0px 0px -5px 0px"), 

53 ), 

54 memory_bar, 

55 ], 

56 layout=widgets.Layout(width="auto", justify_content="center"), 

57 ), 

58 ] 

59 ) 

60 

61 return gpu_status_widget, error_message 

62 

63 

64def monitor_gpu_utilization(utilization_bar, memory_bar): 

65 try: 

66 import pynvml 

67 

68 pynvml.nvmlInit() 

69 handle = pynvml.nvmlDeviceGetHandleByIndex(0) 

70 except Exception: 

71 raise 

72 

73 def update_gpu_status(): 

74 while True: 

75 # cpu_proc = pynvml.nvmlDeviceGetComputeRunningProcesses(handle) 

76 

77 # gpu_procs = pynvml.nvmlDeviceGetGraphicsRunningProcesses(handle) 

78 # num_procs = len(gpu_procs) 

79 

80 # info = pynvml.nvmlDeviceGetMemoryInfo(handle) 

81 # print("Total memory:", info.total) 

82 # print("Free memory:", info.free) 

83 # print("Used memory:", info.used) 

84 

85 res = pynvml.nvmlDeviceGetUtilizationRates(handle) 

86 

87 utilization_bar.value = res.gpu 

88 if res.gpu > 70 and utilization_bar.bar_style == "info": 

89 utilization_bar.bar_style = "warning" 

90 elif res.gpu <= 70 and utilization_bar.bar_style == "warning": 

91 utilization_bar.bar_style = "info" 

92 

93 memory_bar.value = res.memory 

94 if res.memory > 70 and memory_bar.bar_style == "info": 

95 memory_bar.bar_style = "warning" 

96 elif res.memory <= 70 and memory_bar.bar_style == "warning": 

97 memory_bar.bar_style = "info" 

98 

99 time.sleep(1.0) 

100 

101 # update_gpu_status() 

102 thread = threading.Thread(target=update_gpu_status, daemon=True) 

103 thread.start()