diff --git a/getting-started.ipynb b/getting-started.ipynb index 6ed2023..0d00cc4 100644 --- a/getting-started.ipynb +++ b/getting-started.ipynb @@ -1,19 +1,227 @@ { "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# NAS Drives: CMR/SMR, Failure Rates & Specs\n", - "\n", - "Exploring the dataset of 140+ NAS and enterprise hard drives - CMR vs SMR classification, real-world failure rates from Backblaze Drive Stats, and full specs." - ] - }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 1, + "id": "fbe91c0f", + "metadata": { + "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", + "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", + "execution": { + "iopub.execute_input": "2026-06-18T11:22:23.476983Z", + "iopub.status.busy": "2026-06-18T11:22:23.476625Z", + "iopub.status.idle": "2026-06-18T11:22:24.659291Z", + "shell.execute_reply": "2026-06-18T11:22:24.658041Z" + }, + "papermill": { + "duration": 1.189792, + "end_time": "2026-06-18T11:22:24.661531+00:00", + "exception": false, + "start_time": "2026-06-18T11:22:23.471739+00:00", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "142 drives | 3 brands | 39 product lines\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
modelbrandlinecapacity_tbrpmcache_mbinterfaceform_factorrecording_techacoustic_idle_dbacoustic_seek_dbis_heliumdrive_classin_productionafr_pctreliability_drive_countreliability_drive_daysreliability_failuresreliability_source
0WUH722626ALE6L4WDUltrastar DC HC590267200512SATA3.5cmr20.036.01.0Enterprise10.401200.091690.01.0Backblaze 2025
1ST24000NM002HSeagateExos X24247200512SATA3.5cmr30.034.01.0Enterprise12.758328.01941010.0146.0Backblaze 2025
2ST24000NT002SeagateIronWolf Pro247200512SATA3.5cmr28.032.01.0NAS-Pro1NaNNaNNaNNaNNaN
3ST24000VE002SeagateSkyHawk AI247200512SATA3.5cmrNaNNaN1.0Surveillance1NaNNaNNaNNaNNaN
4MG11ACA24TEToshibaMG11247200512SATA3.5cmr20.0NaN1.0Enterprise10.684739.0374087.07.0Backblaze 2025
\n", + "
" + ], + "text/plain": [ + " model brand line capacity_tb rpm cache_mb \\\n", + "0 WUH722626ALE6L4 WD Ultrastar DC HC590 26 7200 512 \n", + "1 ST24000NM002H Seagate Exos X24 24 7200 512 \n", + "2 ST24000NT002 Seagate IronWolf Pro 24 7200 512 \n", + "3 ST24000VE002 Seagate SkyHawk AI 24 7200 512 \n", + "4 MG11ACA24TE Toshiba MG11 24 7200 512 \n", + "\n", + " interface form_factor recording_tech acoustic_idle_db acoustic_seek_db \\\n", + "0 SATA 3.5 cmr 20.0 36.0 \n", + "1 SATA 3.5 cmr 30.0 34.0 \n", + "2 SATA 3.5 cmr 28.0 32.0 \n", + "3 SATA 3.5 cmr NaN NaN \n", + "4 SATA 3.5 cmr 20.0 NaN \n", + "\n", + " is_helium drive_class in_production afr_pct reliability_drive_count \\\n", + "0 1.0 Enterprise 1 0.40 1200.0 \n", + "1 1.0 Enterprise 1 2.75 8328.0 \n", + "2 1.0 NAS-Pro 1 NaN NaN \n", + "3 1.0 Surveillance 1 NaN NaN \n", + "4 1.0 Enterprise 1 0.68 4739.0 \n", + "\n", + " reliability_drive_days reliability_failures reliability_source \n", + "0 91690.0 1.0 Backblaze 2025 \n", + "1 1941010.0 146.0 Backblaze 2025 \n", + "2 NaN NaN NaN \n", + "3 NaN NaN NaN \n", + "4 374087.0 7.0 Backblaze 2025 " + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import pandas as pd\n", "\n", @@ -24,7 +232,17 @@ }, { "cell_type": "markdown", - "metadata": {}, + "id": "b997beaf", + "metadata": { + "papermill": { + "duration": 0.001942, + "end_time": "2026-06-18T11:22:24.665845+00:00", + "exception": false, + "start_time": "2026-06-18T11:22:24.663903+00:00", + "status": "completed" + }, + "tags": [] + }, "source": [ "## CMR vs SMR breakdown\n", "\n", @@ -33,48 +251,551 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 2, + "id": "6a938ae0", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-18T11:22:24.671441Z", + "iopub.status.busy": "2026-06-18T11:22:24.671030Z", + "iopub.status.idle": "2026-06-18T11:22:24.681914Z", + "shell.execute_reply": "2026-06-18T11:22:24.680647Z" + }, + "papermill": { + "duration": 0.016754, + "end_time": "2026-06-18T11:22:24.684488+00:00", + "exception": false, + "start_time": "2026-06-18T11:22:24.667734+00:00", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "recording_tech\n", + "cmr 124\n", + "smr 18\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df['recording_tech'].value_counts()" ] }, { "cell_type": "markdown", - "metadata": {}, + "id": "1bddd132", + "metadata": { + "papermill": { + "duration": 0.002021, + "end_time": "2026-06-18T11:22:24.688888+00:00", + "exception": false, + "start_time": "2026-06-18T11:22:24.686867+00:00", + "status": "completed" + }, + "tags": [] + }, "source": [ "## SMR drives to avoid in a NAS/RAID array" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 3, + "id": "17ee905c", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-18T11:22:24.695808Z", + "iopub.status.busy": "2026-06-18T11:22:24.694779Z", + "iopub.status.idle": "2026-06-18T11:22:24.722063Z", + "shell.execute_reply": "2026-06-18T11:22:24.721040Z" + }, + "papermill": { + "duration": 0.0332, + "end_time": "2026-06-18T11:22:24.724134+00:00", + "exception": false, + "start_time": "2026-06-18T11:22:24.690934+00:00", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
brandlinemodelcapacity_tb
127SeagateBarraCudaST2000DM0082
123SeagateBarraCudaST3000DM0073
109SeagateBarraCudaST4000DM0044
94SeagateBarraCudaST6000DM0036
78SeagateBarraCudaST8000DM0048
139ToshibaL200HDWL1101
140ToshibaMQ04MQ04ABF1001
113ToshibaDT02DT02ABA4004
114ToshibaP300HDWD2404
98ToshibaDT02DT02ABA6006
99ToshibaP300HDWD2606
134WDRedWD20EFAX2
136WDBlueWD20EZAZ2
125WDRedWD30EFAX3
119WDRedWD40EFAX4
122WDBlueWD40EZAZ4
105WDRedWD60EFAX6
108WDBlueWD60EZAZ6
\n", + "
" + ], + "text/plain": [ + " brand line model capacity_tb\n", + "127 Seagate BarraCuda ST2000DM008 2\n", + "123 Seagate BarraCuda ST3000DM007 3\n", + "109 Seagate BarraCuda ST4000DM004 4\n", + "94 Seagate BarraCuda ST6000DM003 6\n", + "78 Seagate BarraCuda ST8000DM004 8\n", + "139 Toshiba L200 HDWL110 1\n", + "140 Toshiba MQ04 MQ04ABF100 1\n", + "113 Toshiba DT02 DT02ABA400 4\n", + "114 Toshiba P300 HDWD240 4\n", + "98 Toshiba DT02 DT02ABA600 6\n", + "99 Toshiba P300 HDWD260 6\n", + "134 WD Red WD20EFAX 2\n", + "136 WD Blue WD20EZAZ 2\n", + "125 WD Red WD30EFAX 3\n", + "119 WD Red WD40EFAX 4\n", + "122 WD Blue WD40EZAZ 4\n", + "105 WD Red WD60EFAX 6\n", + "108 WD Blue WD60EZAZ 6" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[df['recording_tech'] == 'smr'][['brand', 'line', 'model', 'capacity_tb']].sort_values(['brand', 'capacity_tb'])" ] }, { "cell_type": "markdown", - "metadata": {}, + "id": "91573258", + "metadata": { + "papermill": { + "duration": 0.002285, + "end_time": "2026-06-18T11:22:24.728917+00:00", + "exception": false, + "start_time": "2026-06-18T11:22:24.726632+00:00", + "status": "completed" + }, + "tags": [] + }, "source": [ "## Most reliable drives (Backblaze annualized failure rate, lowest first)" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 4, + "id": "fdc66121", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-18T11:22:24.736915Z", + "iopub.status.busy": "2026-06-18T11:22:24.736485Z", + "iopub.status.idle": "2026-06-18T11:22:24.756400Z", + "shell.execute_reply": "2026-06-18T11:22:24.755324Z" + }, + "papermill": { + "duration": 0.027505, + "end_time": "2026-06-18T11:22:24.758809+00:00", + "exception": false, + "start_time": "2026-06-18T11:22:24.731304+00:00", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
brandlinemodelcapacity_tbrecording_techafr_pctreliability_drive_count
79SeagateExos 7E8ST8000NM000A8cmr0.00245.0
0WDUltrastar DC HC590WUH722626ALE6L426cmr0.401200.0
14WDUltrastar DC HC570WUH722222ALE6L422cmr0.4744265.0
32SeagateExos X16ST16000NM001G16cmr0.5434305.0
18ToshibaMG10MG10ACA20TE20cmr0.5917999.0
4ToshibaMG11MG11ACA24TE24cmr0.684739.0
42WDUltrastar DC HC550WUH721816ALE6Lx16cmr0.8029629.0
54WDUltrastar DC HC530WUH721414ALE6L414cmr0.818630.0
38ToshibaMG08MG08ACA16Tx16cmr0.9645652.0
55SeagateExos X16ST12000NM001G12cmr0.9613147.0
43SeagateExos X16ST14000NM001G14cmr1.3510464.0
22SeagateExos X18ST18000NM000J18cmr1.6760.0
39ToshibaMG09MG09ACA16TE16cmr1.73348.0
62WDUltrastar DC HC520HUH721212ALE60412cmr2.6913095.0
1SeagateExos X24ST24000NM002H24cmr2.758328.0
\n", + "
" + ], + "text/plain": [ + " brand line model capacity_tb recording_tech \\\n", + "79 Seagate Exos 7E8 ST8000NM000A 8 cmr \n", + "0 WD Ultrastar DC HC590 WUH722626ALE6L4 26 cmr \n", + "14 WD Ultrastar DC HC570 WUH722222ALE6L4 22 cmr \n", + "32 Seagate Exos X16 ST16000NM001G 16 cmr \n", + "18 Toshiba MG10 MG10ACA20TE 20 cmr \n", + "4 Toshiba MG11 MG11ACA24TE 24 cmr \n", + "42 WD Ultrastar DC HC550 WUH721816ALE6Lx 16 cmr \n", + "54 WD Ultrastar DC HC530 WUH721414ALE6L4 14 cmr \n", + "38 Toshiba MG08 MG08ACA16Tx 16 cmr \n", + "55 Seagate Exos X16 ST12000NM001G 12 cmr \n", + "43 Seagate Exos X16 ST14000NM001G 14 cmr \n", + "22 Seagate Exos X18 ST18000NM000J 18 cmr \n", + "39 Toshiba MG09 MG09ACA16TE 16 cmr \n", + "62 WD Ultrastar DC HC520 HUH721212ALE604 12 cmr \n", + "1 Seagate Exos X24 ST24000NM002H 24 cmr \n", + "\n", + " afr_pct reliability_drive_count \n", + "79 0.00 245.0 \n", + "0 0.40 1200.0 \n", + "14 0.47 44265.0 \n", + "32 0.54 34305.0 \n", + "18 0.59 17999.0 \n", + "4 0.68 4739.0 \n", + "42 0.80 29629.0 \n", + "54 0.81 8630.0 \n", + "38 0.96 45652.0 \n", + "55 0.96 13147.0 \n", + "43 1.35 10464.0 \n", + "22 1.67 60.0 \n", + "39 1.73 348.0 \n", + "62 2.69 13095.0 \n", + "1 2.75 8328.0 " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "df[df['afr_pct'].notna()][['brand', 'line', 'model', 'capacity_tb', 'recording_tech', 'afr_pct', 'reliability_drive_count']].sort_values('afr_pct')" ] }, { "cell_type": "markdown", - "metadata": {}, + "id": "5d1cd51f", + "metadata": { + "papermill": { + "duration": 0.002634, + "end_time": "2026-06-18T11:22:24.764055+00:00", + "exception": false, + "start_time": "2026-06-18T11:22:24.761421+00:00", + "status": "completed" + }, + "tags": [] + }, "source": [ "## Data & API\n", "\n", @@ -85,16 +806,45 @@ } ], "metadata": { + "kaggle": { + "accelerator": "none", + "dataSources": [], + "dockerImageVersionId": 28755, + "isGpuEnabled": false, + "isInternetEnabled": false, + "language": "python", + "sourceType": "notebook" + }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.12.0" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 5.557784, + "end_time": "2026-06-18T11:22:25.388506+00:00", + "environment_variables": {}, + "exception": null, + "input_path": "__notebook__.ipynb", + "output_path": "__notebook__.ipynb", + "parameters": {}, + "start_time": "2026-06-18T11:22:19.830722+00:00", + "version": "2.7.0" } }, "nbformat": 4, - "nbformat_minor": 4 + "nbformat_minor": 5 }