@mhg/blog
Version:
1 lines • 267 kB
HTML
<!DOCTYPE html><html lang="zh-CN"><head hexo-theme="https://github.com/volantis-x/hexo-theme-volantis/#6.0.0-alpha.0"><meta name="generator" content="Hexo 6.3.0"><meta name="Volantis" content="6.0.0-alpha.0"><meta charset="utf-8"><link rel="canonical" href="https://blog.mhuig.top/notes/spark/k-means.html"><meta http-equiv="x-dns-prefetch-control" content="on"><link rel="dns-prefetch" href="https://static.mhuig.top"><link rel="preconnect" href="https://static.mhuig.top" crossorigin><meta name="renderer" content="webkit"><meta name="force-rendering" content="webkit"><meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1"><meta name="HandheldFriendly" content="True"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"><meta content="black-translucent" name="apple-mobile-web-app-status-bar-style"><meta content="telephone=no" name="format-detection"><script>function registerServiceWorker(e){"localhost"!=window.location.hostname&&"serviceWorker"in navigator&&navigator.serviceWorker.register(`${e}`).then((function(e){"localhost"==window.location.hostname&&(e.onupdatefound=function(){var r=e.installing;r.onstatechange=function(){switch(r.state){case"installed":navigator.serviceWorker.controller?console.log("Updated Service Worker."):console.log("Service Worker Sucess!");break;case"redundant":console.log("The installing service worker became redundant.")}}})})).catch((function(e){console.error("Error during service worker registration:",e),"undefined"==typeof swinstallretry&&(swinstallretry=1,registerServiceWorker("/jquery.js?v=0258ff4f79"))}))}registerServiceWorker("/jquery.js?v=0258ff4f79")</script><link rel="apple-touch-icon" sizes="180x180" href="/lib/favicon/apple-touch-icon.png?v=e035908f18"><link rel="icon" type="image/png" sizes="32x32" href="/lib/favicon/favicon-32x32.png?v=b9d9ae54cc"><link rel="icon" type="image/png" sizes="192x192" href="/lib/favicon/android-chrome-192x192.png?v=8bf02188e3"><link rel="icon" type="image/png" sizes="144x144" href="/lib/favicon/android-chrome-144x144.png?v=5c8eadcbb7"><link rel="icon" type="image/png" sizes="16x16" href="/lib/favicon/favicon-16x16.png?v=f896f2f634"><link rel="manifest" href="/lib/favicon/site.webmanifest?v=f138de5899"><link rel="mask-icon" href="/lib/favicon/safari-pinned-tab.svg?v=87d61e51a5" color="#5bbad5"><meta name="apple-mobile-web-app-title" content="MHuiG's Blog"><meta name="application-name" content="MHuiG's Blog"><meta name="msapplication-TileColor" content="#87ceeb"><meta name="msapplication-TileImage" content="/lib/favicon/mstile-144x144.png"><meta name="theme-color" content="#87ceeb" media="(prefers-color-scheme: light)"><meta name="theme-color" content="#21232f" media="(prefers-color-scheme: dark)"><link href="/opensearch.xml?v=2f4fd4e7e0" rel="search" title="MHuiG's Blog" type="application/opensearchdescription+xml"><link rel="preload" href="/css/style.css?v=2d24f6fe3e" as="style"><link rel="preload" href="https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/fonts/VarelaRound/VarelaRound-Regular.ttf" as="font" type="font/ttf" crossorigin="anonymous"><link rel="alternate" href="/atom.xml?v=fe8205151a" title="MHuiG" type="application/atom+xml"><link rel="alternate" href="/rss2.xml?v=6b8706d024" title="MHuiG" type="application/rss+xml"><title>Spark: 基于K均值聚类的网络流量异常检测(pyspark) - MHuiG</title><meta name="keywords" content="Blog, IT, BigData, MHuiG, @MHuiG, iMHuiG, 博客, 梦幻岛, Neverland, 技术博客"><meta desc name="description" content="基于K均值聚类的网络流量异常检测(pyspark) 异常检测常用于检测欺诈、网络攻击、服务器及传感设备故障。在这些应用中，我们要能够找出以前从未见过的新型异常，如新欺诈方式、新入侵方法或新服务器故障模式。 - MHuiG - MHuiG"><meta property="og:type" content="website"><meta property="og:title" content="MHuiG"><meta property="og:url" content="https://blog.mhuig.top/notes/Spark/k-means"><meta property="og:site_name" content="MHuiG"><meta property="og:description" content="基于K均值聚类的网络流量异常检测(pyspark) 异常检测常用于检测欺诈、网络攻击、服务器及传感设备故障。在这些应用中，我们要能够找出以前从未见过的新型异常，如新欺诈方式、新入侵方法或新服务器故障模式。"><meta property="og:locale" content="zh_CN"><meta property="og:image" content="https://blog.mhuig.top/lib/favicon/android-chrome-192x192.png"><meta property="article:published_time" content="2020-08-02T10:23:00.000Z"><meta property="article:modified_time" content="2022-05-10T02:11:00.000Z"><meta property="article:author" content="MHuiG"><meta property="article:tag" content="Spark"><meta name="twitter:card" content="summary"><meta name="twitter:image" content="https://blog.mhuig.top/lib/favicon/android-chrome-192x192.png"><meta name="twitter:creator" content="@iMHuiG"><meta name="twitter:site" content="@iMHuiG"><style>#safearea{display:none}.post-story+.post-story{content-visibility:auto;contain-intrinsic-size:10px 500px}:root{--color-site-body:#87ceeb;--color-site-bg:#87ceeb;--color-site-inner:#555;--color-site-footer:#666;--color-card:#fff;--color-text:#444;--color-block:#f6f6f6;--color-inlinecode:#d56d28;--color-codeblock:#fff7ea;--color-h1:#3a3a3a;--color-h2:#3a3a3a;--color-h3:#333;--color-h4:#444;--color-h5:#555;--color-h6:#666;--color-p:#444;--color-list:#666;--color-list-hl:#1a78c2;--color-meta:#888;--color-read-bkg:#e0d8c8;--color-read-post:#f8f1e2;--color-copyright-bkg:#f5f5f5}*{box-sizing:border-box;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;outline:0;margin:0;padding:0}::-webkit-scrollbar{height:4px;width:4px}::-webkit-scrollbar-track-piece{background:0 0}::-webkit-scrollbar-thumb{background:#2196f3;cursor:pointer;border-radius:2px;-webkit-border-radius:2px}::-webkit-scrollbar-thumb:hover{background:#ff5722}html{color:var(--color-text);width:100%;height:100%;font-family:"Varela Round","PingFang SC","Microsoft YaHei",Helvetica,Arial,Menlo,Monaco,monospace,sans-serif;font-size:16px}html>::-webkit-scrollbar{height:4px;width:4px}html>::-webkit-scrollbar-track-piece{background:0 0}html>::-webkit-scrollbar-thumb{background:#2196f3;cursor:pointer;border-radius:2px;-webkit-border-radius:2px}html>::-webkit-scrollbar-thumb:hover{background:#ff5722}body{background-color:var(--color-site-body);text-rendering:optimizelegibility;-webkit-tap-highlight-color:transparent;line-height:1.6;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}body.modal-active{overflow:hidden}@media screen and (max-width:680px){body.modal-active{position:fixed;top:0;right:0;bottom:0;left:0}}a{color:#2196f3;cursor:pointer;text-decoration:none;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}a:hover{color:#ff5722}a:active,a:hover{outline:0}ol,ul{padding-left:0}ol li,ul li{list-style:none}header{display:-webkit-box;display:-moz-box;display:block}img{border:0;background:0 0;max-width:100%}svg:not(:root){overflow:hidden}hr{-moz-box-sizing:content-box;box-sizing:content-box;-webkit-box-sizing:content-box;-moz-box-sizing:content-box;height:0;border:0;border-radius:1px;-webkit-border-radius:1px;border-bottom:1px solid rgba(68,68,68,.1)}button,input{color:inherit;font:inherit;margin:0}button{overflow:visible;text-transform:none;-webkit-appearance:button;cursor:pointer}@supports (backdrop-filter:blur(20px)){.blur{background:rgba(255,255,255,.9)!important;backdrop-filter:saturate(200%) blur(20px)}}.shadow{box-shadow:0 1px 2px 0 rgba(0,0,0,.1);-webkit-box-shadow:0 1px 2px 0 rgba(0,0,0,.1)}.shadow.floatable{transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}.shadow.floatable:hover{box-shadow:0 2px 4px 0 rgba(0,0,0,.1),0 4px 8px 0 rgba(0,0,0,.1),0 8px 16px 0 rgba(0,0,0,.1);-webkit-box-shadow:0 2px 4px 0 rgba(0,0,0,.1),0 4px 8px 0 rgba(0,0,0,.1),0 8px 16px 0 rgba(0,0,0,.1)}#l_cover{min-height:64px}.cover-wrapper{top:0;left:0;max-width:100%;height:100vh;display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;flex-wrap:nowrap;-webkit-flex-wrap:nowrap;-khtml-flex-wrap:nowrap;-moz-flex-wrap:nowrap;-o-flex-wrap:nowrap;-ms-flex-wrap:nowrap;-webkit-box-direction:normal;-moz-box-direction:normal;-webkit-box-orient:vertical;-moz-box-orient:vertical;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;align-items:center;align-self:center;align-content:center;color:var(--color-site-inner);padding:0 16px;user-select:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;position:relative;overflow:hidden;margin-bottom:-100px}.cover-wrapper .cover-bg{position:absolute;width:100%;height:100%;background-position:center;background-size:cover;-webkit-background-size:cover;-moz-background-size:cover}.cover-wrapper .cover-bg.lazyload:not(.loaded){opacity:0;-webkit-opacity:0;-moz-opacity:0}.cover-wrapper .cover-bg.lazyload.loaded{animation-delay:0s;animation-duration:.5s;animation-fill-mode:forwards;animation-timing-function:ease-out;animation-name:fadeIn}@-moz-keyframes fadeIn{0%{opacity:0;-webkit-opacity:0;-moz-opacity:0;filter:blur(12px);transform:scale(1.02);-webkit-transform:scale(1.02);-khtml-transform:scale(1.02);-moz-transform:scale(1.02);-o-transform:scale(1.02);-ms-transform:scale(1.02)}100%{opacity:1;-webkit-opacity:1;-moz-opacity:1}}@-webkit-keyframes fadeIn{0%{opacity:0;-webkit-opacity:0;-moz-opacity:0;filter:blur(12px);transform:scale(1.02);-webkit-transform:scale(1.02);-khtml-transform:scale(1.02);-moz-transform:scale(1.02);-o-transform:scale(1.02);-ms-transform:scale(1.02)}100%{opacity:1;-webkit-opacity:1;-moz-opacity:1}}@-o-keyframes fadeIn{0%{opacity:0;-webkit-opacity:0;-moz-opacity:0;filter:blur(12px);transform:scale(1.02);-webkit-transform:scale(1.02);-khtml-transform:scale(1.02);-moz-transform:scale(1.02);-o-transform:scale(1.02);-ms-transform:scale(1.02)}100%{opacity:1;-webkit-opacity:1;-moz-opacity:1}}@keyframes fadeIn{0%{opacity:0;-webkit-opacity:0;-moz-opacity:0;filter:blur(12px);transform:scale(1.02);-webkit-transform:scale(1.02);-khtml-transform:scale(1.02);-moz-transform:scale(1.02);-o-transform:scale(1.02);-ms-transform:scale(1.02)}100%{opacity:1;-webkit-opacity:1;-moz-opacity:1}}.cover-wrapper .cover-body{z-index:1;position:relative;width:100%;height:100%}.cover-wrapper#full{height:calc(100vh + 100px);padding-bottom:100px}.cover-wrapper#half{max-height:640px;min-height:400px;height:calc(36vh - 64px + 200px)}.cover-wrapper #scroll-down{width:100%;height:64px;position:absolute;bottom:100px;text-align:center;cursor:pointer}.cover-wrapper #scroll-down .scroll-down-effects{color:#fff;font-size:24px;line-height:64px;position:absolute;width:24px;left:calc(50% - 12px);text-shadow:0 1px 2px rgba(0,0,0,.1);animation:scroll-down-effect 1.5s infinite;-webkit-animation:scroll-down-effect 1.5s infinite;-khtml-animation:scroll-down-effect 1.5s infinite;-moz-animation:scroll-down-effect 1.5s infinite;-o-animation:scroll-down-effect 1.5s infinite;-ms-animation:scroll-down-effect 1.5s infinite}@-moz-keyframes scroll-down-effect{0%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}50%{top:-16px;opacity:.4;-webkit-opacity:0.4;-moz-opacity:0.4}100%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}}@-webkit-keyframes scroll-down-effect{0%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}50%{top:-16px;opacity:.4;-webkit-opacity:0.4;-moz-opacity:0.4}100%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}}@-o-keyframes scroll-down-effect{0%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}50%{top:-16px;opacity:.4;-webkit-opacity:0.4;-moz-opacity:0.4}100%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}}@keyframes scroll-down-effect{0%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}50%{top:-16px;opacity:.4;-webkit-opacity:0.4;-moz-opacity:0.4}100%{top:0;opacity:1;-webkit-opacity:1;-moz-opacity:1}}.cover-wrapper .cover-body{margin-top:64px;margin-bottom:100px}.cover-wrapper .cover-body,.cover-wrapper .cover-body .bottom,.cover-wrapper .cover-body .top{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;-webkit-box-direction:normal;-moz-box-direction:normal;-webkit-box-orient:vertical;-moz-box-orient:vertical;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;align-items:center;justify-content:center;-webkit-justify-content:center;-khtml-justify-content:center;-moz-justify-content:center;-o-justify-content:center;-ms-justify-content:center;max-width:100%}.cover-wrapper .cover-body .bottom{margin-top:32px}.cover-wrapper .cover-body .title{font-family:"Varela Round","PingFang SC","Microsoft YaHei",Helvetica,Arial,Helvetica,monospace;font-size:3.125rem;line-height:1.2;text-shadow:0 1px 2px rgba(0,0,0,.1)}.cover-wrapper .cover-body .subtitle{font-size:20px}.cover-wrapper .cover-body .logo{max-height:120px;max-width:calc(100% - 4 * 16px)}@media screen and (min-height:1024px){.cover-wrapper .cover-body .title{font-size:3rem}.cover-wrapper .cover-body .subtitle{font-size:1.05rem}.cover-wrapper .cover-body .logo{max-height:150px}}.cover-wrapper .cover-body .m_search{position:relative;max-width:calc(100% - 16px);width:320px;vertical-align:middle}.cover-wrapper .cover-body .m_search .form{position:relative;display:-webkit-box;display:-moz-box;display:block;width:100%}.cover-wrapper .cover-body .m_search .icon,.cover-wrapper .cover-body .m_search .input{transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}.cover-wrapper .cover-body .m_search .icon{position:absolute;display:-webkit-box;display:-moz-box;display:block;line-height:2.5rem;width:32px;top:0;left:5px;color:rgba(68,68,68,.75)}.cover-wrapper .cover-body .m_search .input{display:-webkit-box;display:-moz-box;display:block;height:2.5rem;width:100%;box-shadow:none;-webkit-box-shadow:none;box-sizing:border-box;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;font-size:.875rem;-webkit-appearance:none;padding-left:36px;border-radius:1.4rem;-webkit-border-radius:1.4rem;background:rgba(255,255,255,.6);backdrop-filter:blur(10px);border:none;color:var(--color-text)}@media screen and (max-width:500px){.cover-wrapper .cover-body .m_search .input{padding-left:36px}}.cover-wrapper .cover-body .m_search .input:hover{background:rgba(255,255,255,.8)}.cover-wrapper .cover-body .m_search .input:focus{background:#fff}.cover-wrapper .list-h{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;-webkit-box-direction:normal;-moz-box-direction:normal;-webkit-box-orient:horizontal;-moz-box-orient:horizontal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;flex-wrap:wrap;-webkit-flex-wrap:wrap;-khtml-flex-wrap:wrap;-moz-flex-wrap:wrap;-o-flex-wrap:wrap;-ms-flex-wrap:wrap;align-items:stretch;border-radius:4px;-webkit-border-radius:4px;user-select:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none}.cover-wrapper .list-h a{-webkit-box-flex:1;-moz-box-flex:1;-webkit-flex:1 0;-ms-flex:1 0;flex:1 0;display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;font-weight:600}.cover-wrapper .list-h a img{display:-webkit-box;display:-moz-box;display:block;border-radius:2px;-webkit-border-radius:2px;margin:4px;min-width:40px;max-width:44px}@media screen and (max-width:768px){.cover-wrapper .list-h a img{min-width:36px;max-width:40px}}@media screen and (max-width:500px){.cover-wrapper .list-h a img{margin:2px 4px;min-width:32px;max-width:36px}}@media screen and (max-width:375px){.cover-wrapper .list-h a img{min-width:28px;max-width:32px}}.cover-wrapper{max-width:100%}.cover-wrapper.search .bottom .menu{margin-top:16px}.cover-wrapper.search .bottom .menu .list-h a{white-space:nowrap;-webkit-box-direction:normal;-moz-box-direction:normal;-webkit-box-orient:horizontal;-moz-box-orient:horizontal;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;align-items:baseline;padding:2px;margin:4px;color:var(--color-site-inner);opacity:.75;-webkit-opacity:0.75;-moz-opacity:0.75;text-shadow:0 1px 2px rgba(0,0,0,.05);border-bottom:2px solid transparent}.cover-wrapper.search .bottom .menu .list-h a i{margin-right:4px}.cover-wrapper.search .bottom .menu .list-h a p{font-size:.9375rem}.cover-wrapper.search .bottom .menu .list-h a.active,.cover-wrapper.search .bottom .menu .list-h a:active,.cover-wrapper.search .bottom .menu .list-h a:hover{opacity:1;-webkit-opacity:1;-moz-opacity:1;border-bottom:2px solid var(--color-site-inner)}.cover-wrapper.dock .menu,.cover-wrapper.featured .menu,.cover-wrapper.focus .menu{border-radius:6px;-webkit-border-radius:6px}.cover-wrapper.dock .menu .list-h a,.cover-wrapper.featured .menu .list-h a,.cover-wrapper.focus .menu .list-h a{-webkit-box-direction:normal;-moz-box-direction:normal;-webkit-box-orient:vertical;-moz-box-orient:vertical;-webkit-flex-direction:column;-ms-flex-direction:column;flex-direction:column;align-items:center;padding:12px;line-height:24px;border-radius:4px;-webkit-border-radius:4px;border-bottom:none;text-align:center;align-content:flex-end;color:rgba(68,68,68,.7);font-size:1.5rem}@media screen and (max-width:500px){.cover-wrapper.dock .menu .list-h a,.cover-wrapper.featured .menu .list-h a,.cover-wrapper.focus .menu .list-h a{padding:12px 8px}}.cover-wrapper.dock .menu .list-h a i,.cover-wrapper.featured .menu .list-h a i,.cover-wrapper.focus .menu .list-h a i{margin:8px}.cover-wrapper.dock .menu .list-h a p,.cover-wrapper.featured .menu .list-h a p,.cover-wrapper.focus .menu .list-h a p{font-size:.875rem}.cover-wrapper.dock .menu .list-h a.active,.cover-wrapper.featured .menu .list-h a.active,.cover-wrapper.focus .menu .list-h a.active{background:var(--color-card);backdrop-filter:none}.cover-wrapper.dock .menu .list-h a.active i,.cover-wrapper.dock .menu .list-h a.active i+p,.cover-wrapper.featured .menu .list-h a.active i,.cover-wrapper.featured .menu .list-h a.active i+p,.cover-wrapper.focus .menu .list-h a.active i,.cover-wrapper.focus .menu .list-h a.active i+p{color:#2196f3}.cover-wrapper.dock .menu .list-h a.active img+p,.cover-wrapper.featured .menu .list-h a.active img+p,.cover-wrapper.focus .menu .list-h a.active img+p{color:var(--color-text)}.cover-wrapper.dock .menu .list-h a:hover,.cover-wrapper.featured .menu .list-h a:hover,.cover-wrapper.focus .menu .list-h a:hover{background:var(--color-card)}.cover-wrapper.featured .menu .list-h{margin:-2px}.cover-wrapper.featured .menu .list-h a{margin:2px;background:rgba(255,255,255,.5)}@supports (backdrop-filter:blur(20px)){.cover-wrapper.featured .menu .list-h a{background:rgba(255,255,255,.5);backdrop-filter:saturate(200%) blur(20px)}}@media (prefers-color-scheme:dark){:root{--color-mode:'dark'}:root:not([color-scheme]){--color-site-body:#121212;--color-read-bkg:#21232f;--color-read-post:#252d38;--color-site-bg:#21232f;--color-site-inner:#efefef;--color-site-footer:#666;--color-card:#252d38;--color-text:rgba(238,238,238,0.871);--color-block:rgba(68,68,68,0.65);--color-codeblock:rgba(68,68,68,0.65);--color-inlinecode:#d56d28;--color-h1:rgba(255,255,255,0.871);--color-h2:rgba(255,255,255,0.871);--color-h3:rgba(255,255,255,0.6);--color-h4:rgba(255,255,255,0.6);--color-h5:rgba(255,255,255,0.6);--color-h6:rgba(255,255,255,0.6);--color-p:rgba(217,217,217,0.871);--color-list:rgba(217,217,217,0.871);--color-list-hl:#4dabf5;--color-meta:rgba(191,191,191,0.871);--color-link:rgba(191,191,191,0.871);--color-copyright-bkg:#21252b}:root:not([color-scheme]) img{filter:brightness(70%)!important}:root:not([color-scheme]) .blur{background:rgba(33,35,47,.9)!important}:root:not([color-scheme]) .white-box.blur{background:rgba(37,45,56,.9)!important}:root:not([color-scheme]) .nav-main .u-search-input{background:var(--color-card)!important}:root:not([color-scheme]) #l_main .article .prev-next>a{background:var(--color-block)!important}:root:not([color-scheme]) #l_main .article .prev-next>a:hover{background:var(--color-site-bg)!important}:root:not([color-scheme]) .article blockquote{background:var(--color-block)!important}:root:not([color-scheme]) .article-title a{color:var(--color-h1)!important}:root:not([color-scheme]) details>summary{color:var(--color-p)!important;background:var(--color-site-bg)!important}:root:not([color-scheme]) details{border:1px solid var(--color-site-bg)!important;background:var(--color-site-bg)!important}:root:not([color-scheme]) #u-search .modal,:root:not([color-scheme]) #u-search .modal-body,:root:not([color-scheme]) #u-search .modal-header{background:var(--color-card)!important}:root:not([color-scheme]) #u-search .modal-body .modal-results .result:hover{background:var(--color-block)!important}:root:not([color-scheme]) .u-search-input:hover{background:var(--color-block)!important}:root:not([color-scheme]) .u-search-input:focus{background:var(--color-site-body)!important}}[color-scheme=dark]{--color-site-body:#121212;--color-read-bkg:#21232f;--color-read-post:#252d38;--color-site-bg:#21232f;--color-site-inner:#efefef;--color-site-footer:#666;--color-card:#252d38;--color-text:rgba(238,238,238,0.871);--color-block:rgba(68,68,68,0.65);--color-codeblock:rgba(68,68,68,0.65);--color-inlinecode:#d56d28;--color-h1:rgba(255,255,255,0.871);--color-h2:rgba(255,255,255,0.871);--color-h3:rgba(255,255,255,0.6);--color-h4:rgba(255,255,255,0.6);--color-h5:rgba(255,255,255,0.6);--color-h6:rgba(255,255,255,0.6);--color-p:rgba(217,217,217,0.871);--color-list:rgba(217,217,217,0.871);--color-list-hl:#4dabf5;--color-meta:rgba(191,191,191,0.871);--color-link:rgba(191,191,191,0.871);--color-copyright-bkg:#21252b}[color-scheme=dark] img{filter:brightness(70%)!important}[color-scheme=dark] .blur{background:rgba(33,35,47,.9)!important}[color-scheme=dark] .white-box.blur{background:rgba(37,45,56,.9)!important}[color-scheme=dark] .nav-main .u-search-input{background:var(--color-card)!important}[color-scheme=dark] #l_main .article .prev-next>a{background:var(--color-block)!important}[color-scheme=dark] #l_main .article .prev-next>a:hover{background:var(--color-site-bg)!important}[color-scheme=dark] .article blockquote{background:var(--color-block)!important}[color-scheme=dark] .article-title a{color:var(--color-h1)!important}[color-scheme=dark] details>summary{color:var(--color-p)!important;background:var(--color-site-bg)!important}[color-scheme=dark] details{border:1px solid var(--color-site-bg)!important;background:var(--color-site-bg)!important}[color-scheme=dark] #u-search .modal,[color-scheme=dark] #u-search .modal-body,[color-scheme=dark] #u-search .modal-header{background:var(--color-card)!important}[color-scheme=dark] #u-search .modal-body .modal-results .result:hover{background:var(--color-block)!important}[color-scheme=dark] .u-search-input:hover{background:var(--color-block)!important}[color-scheme=dark] .u-search-input:focus{background:var(--color-site-body)!important}@media screen and (max-width:500px){[color-scheme=dark] .l_header .m_search{background:var(--color-site-bg)!important}}@font-face{font-family:'Varela Round';src:url("");font-weight:'normal';font-style:'normal';font-display:swap}@font-face{font-family:'Varela Round';src:url("https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/fonts/VarelaRound/VarelaRound-Regular.ttf");font-weight:'normal';font-style:'normal';font-display:swap}.l_header{position:fixed;z-index:1000;top:0;width:100%;height:64px;background:var(--color-card);box-shadow:0 1px 2px 0 rgba(0,0,0,.1);-webkit-box-shadow:0 1px 2px 0 rgba(0,0,0,.1)}.l_header.auto{transition:opacity .4s ease;-webkit-transition:opacity .4s ease;-khtml-transition:opacity 0.4s ease;-moz-transition:opacity .4s ease;-o-transition:opacity .4s ease;-ms-transition:opacity .4s ease;visibility:hidden}.l_header.auto.show{opacity:1!important;-webkit-opacity:1!important;-moz-opacity:1!important;visibility:visible}.l_header .container{margin-left:16px;margin-right:16px}.l_header #wrapper{height:100%;user-select:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none}.l_header #wrapper .nav-main,.l_header #wrapper .nav-sub{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;flex-wrap:nowrap;-webkit-flex-wrap:nowrap;-khtml-flex-wrap:nowrap;-moz-flex-wrap:nowrap;-o-flex-wrap:nowrap;-ms-flex-wrap:nowrap;justify-content:space-between;-webkit-justify-content:space-between;-khtml-justify-content:space-between;-moz-justify-content:space-between;-o-justify-content:space-between;-ms-justify-content:space-between;align-items:center}.l_header #wrapper .nav-main{transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}.l_header #wrapper.sub .nav-main{transform:translateY(-64px);-webkit-transform:translateY(-64px);-khtml-transform:translateY(-64px);-moz-transform:translateY(-64px);-o-transform:translateY(-64px);-ms-transform:translateY(-64px)}.l_header #wrapper .nav-sub{transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease;opacity:0;-webkit-opacity:0;-moz-opacity:0;height:64px;width:calc(100% - 2 * 16px);position:absolute}.l_header #wrapper .nav-sub ::-webkit-scrollbar{display:-webkit-box;display:-moz-box;display:none}@media screen and (min-width:2048px){.l_header #wrapper .nav-sub{max-width:55vw;margin:auto}}.l_header #wrapper.sub .nav-sub{opacity:1;-webkit-opacity:1;-moz-opacity:1}.l_header #wrapper .title{position:relative;color:var(--color-text);padding-left:24px;max-height:64px}.l_header #wrapper .nav-main .title{white-space:nowrap;overflow:hidden;text-overflow:ellipsis;flex-shrink:0;line-height:64px;padding:0 24px;font-size:1.25rem;font-family:"Varela Round","PingFang SC","Microsoft YaHei",Helvetica,Arial,Helvetica,monospace}.l_header #wrapper .nav-main .title img{height:64px}.l_header .nav-sub{max-width:1080px;margin:auto}.l_header .nav-sub .title{font-weight:700;font-family:"Varela Round","PingFang SC","Microsoft YaHei",Helvetica,Arial,Menlo,Monaco,monospace,sans-serif;line-height:1.2;max-height:64px;white-space:normal;flex-shrink:1}.l_header .switcher{display:-webkit-box;display:-moz-box;display:none;line-height:64px;align-items:center}.l_header .switcher .s-toc{display:-webkit-box;display:-moz-box;display:none}@media screen and (max-width:768px){.l_header .switcher .s-toc{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex}}.l_header .switcher>li{height:48px;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease;margin:2px}@media screen and (max-width:500px){.l_header .switcher>li{margin:0 1px;height:48px}}.l_header .switcher>li>a{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;justify-content:center;-webkit-justify-content:center;-khtml-justify-content:center;-moz-justify-content:center;-o-justify-content:center;-ms-justify-content:center;align-items:center;width:48px;height:48px;padding:.85em 1.1em;border-radius:100px;-webkit-border-radius:100px;border:none;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease;color:#2196f3}.l_header .switcher>li>a:hover{border:none}.l_header .switcher>li>a.active,.l_header .switcher>li>a:active{border:none;background:var(--color-site-bg)}@media screen and (max-width:500px){.l_header .switcher>li>a{width:36px;height:48px}}.l_header .nav-sub .switcher{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex}.l_header .m_search{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;height:64px;width:240px;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}@media screen and (max-width:1024px){.l_header .m_search{width:44px;min-width:44px}.l_header .m_search input::placeholder{opacity:0;-webkit-opacity:0;-moz-opacity:0}.l_header .m_search:hover{width:240px}.l_header .m_search:hover input::placeholder{opacity:1;-webkit-opacity:1;-moz-opacity:1}}@media screen and (min-width:500px){.l_header .m_search:hover .input{width:100%}.l_header .m_search:hover .input::placeholder{opacity:1;-webkit-opacity:1;-moz-opacity:1}}@media screen and (max-width:500px){.l_header .m_search{min-width:0}.l_header .m_search input::placeholder{opacity:1;-webkit-opacity:1;-moz-opacity:1}}.l_header .m_search .form{position:relative;display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;width:100%;align-items:center}.l_header .m_search .icon{position:absolute;width:36px;left:5px;color:var(--color-meta)}@media screen and (max-width:500px){.l_header .m_search .icon{display:-webkit-box;display:-moz-box;display:none}}.l_header .m_search .input{display:-webkit-box;display:-moz-box;display:block;padding-top:8px;padding-bottom:8px;line-height:1.3;width:100%;color:var(--color-text);background:#fafafa;box-shadow:none;-webkit-box-shadow:none;box-sizing:border-box;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;padding-left:40px;font-size:.875rem;border-radius:8px;-webkit-border-radius:8px;border:none;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}@media screen and (min-width:500px){.l_header .m_search .input:focus{box-shadow:0 4px 8px 0 rgba(0,0,0,.1);-webkit-box-shadow:0 4px 8px 0 rgba(0,0,0,.1)}}@media screen and (max-width:500px){.l_header .m_search .input{background:var(--color-block);padding-left:8px;border:none}.l_header .m_search .input:focus,.l_header .m_search .input:hover{border:none}}@media (max-width:500px){.l_header .m_search{left:0;width:0;overflow:hidden;position:absolute;background:#fff;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}.l_header .m_search .input{border-radius:32px;-webkit-border-radius:32px;margin-left:16px;padding-left:16px}.l_header.z_search-open .m_search{width:100%}.l_header.z_search-open .m_search .input{width:calc(100% - 120px)}}ul.m-pc>li>a{color:inherit;border-bottom:2px solid transparent}ul.m-pc>li>a.active,ul.m-pc>li>a:active{border-bottom:2px solid #2196f3}ul.list-v li:hover>ul.list-v,ul.m-pc li:hover>ul.list-v{display:-webkit-box;display:-moz-box;display:block}ul.nav-list-h{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;align-items:stretch}ul.nav-list-h>li{position:relative;justify-content:center;-webkit-justify-content:center;-khtml-justify-content:center;-moz-justify-content:center;-o-justify-content:center;-ms-justify-content:center;height:100%;line-height:2.4;border-radius:4px;-webkit-border-radius:4px}ul.nav-list-h>li>a{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;font-weight:600}ul.list-v{z-index:1;display:-webkit-box;display:-moz-box;display:none;position:absolute;background:var(--color-card);box-shadow:0 2px 4px 0 rgba(0,0,0,.08),0 4px 8px 0 rgba(0,0,0,.08),0 8px 16px 0 rgba(0,0,0,.08);-webkit-box-shadow:0 2px 4px 0 rgba(0,0,0,.08),0 4px 8px 0 rgba(0,0,0,.08),0 8px 16px 0 rgba(0,0,0,.08);margin-top:-6px;border-radius:4px;-webkit-border-radius:4px;padding:8px 0}ul.list-v.show{display:-webkit-box;display:-moz-box;display:block}ul.list-v hr{margin-top:8px;margin-bottom:8px}ul.list-v>li{white-space:nowrap;word-break:keep-all}ul.list-v>li.header{font-size:.78125rem;font-weight:700;line-height:2em;color:var(--color-meta);margin:8px 16px 4px}ul.list-v>li.header i{margin-right:8px}ul.list-v>li ul{margin-left:0;display:-webkit-box;display:-moz-box;display:none;margin-top:-40px}ul.list-v .aplayer-container{min-height:64px;padding:6px 16px}ul.list-v>li>a{transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease;display:-webkit-box;display:-moz-box;display:block;color:var(--color-list);font-size:.875rem;font-weight:700;line-height:36px;padding:0 20px 0 16px;text-overflow:ellipsis;margin:0 4px;border-radius:4px;-webkit-border-radius:4px}@media screen and (max-width:1024px){ul.list-v>li>a{line-height:40px}}ul.list-v>li>a>i{margin-right:8px}ul.list-v>li>a.active,ul.list-v>li>a:active{color:var(--color-list-hl)}ul.list-v>li>a:hover{color:var(--color-list-hl);background:var(--color-site-bg)}.l_header .menu>ul>li>a{display:-webkit-box;display:-moz-box;display:block;padding:0 8px}.l_header .menu>ul>li>a>i{margin-right:4px}.l_header ul.nav-list-h>li{color:var(--color-list);line-height:64px}.l_header ul.nav-list-h>li>a{max-height:64px;overflow:hidden;color:inherit}.l_header ul.nav-list-h>li>a.active,.l_header ul.nav-list-h>li>a:active{color:#2196f3}.l_header ul.nav-list-h>li:hover>a{color:var(--color-list-hl)}.l_header ul.nav-list-h>li i.music{animation:rotate-effect 1.5s linear infinite;-webkit-animation:rotate-effect 1.5s linear infinite;-khtml-animation:rotate-effect 1.5s linear infinite;-moz-animation:rotate-effect 1.5s linear infinite;-o-animation:rotate-effect 1.5s linear infinite;-ms-animation:rotate-effect 1.5s linear infinite}@-moz-keyframes rotate-effect{0%{transform:rotate(0);-webkit-transform:rotate(0);-khtml-transform:rotate(0);-moz-transform:rotate(0);-o-transform:rotate(0);-ms-transform:rotate(0)}25%{transform:rotate(90deg);-webkit-transform:rotate(90deg);-khtml-transform:rotate(90deg);-moz-transform:rotate(90deg);-o-transform:rotate(90deg);-ms-transform:rotate(90deg)}50%{transform:rotate(180deg);-webkit-transform:rotate(180deg);-khtml-transform:rotate(180deg);-moz-transform:rotate(180deg);-o-transform:rotate(180deg);-ms-transform:rotate(180deg)}75%{transform:rotate(270deg);-webkit-transform:rotate(270deg);-khtml-transform:rotate(270deg);-moz-transform:rotate(270deg);-o-transform:rotate(270deg);-ms-transform:rotate(270deg)}100%{transform:rotate(360deg);-webkit-transform:rotate(360deg);-khtml-transform:rotate(360deg);-moz-transform:rotate(360deg);-o-transform:rotate(360deg);-ms-transform:rotate(360deg)}}@-webkit-keyframes rotate-effect{0%{transform:rotate(0);-webkit-transform:rotate(0);-khtml-transform:rotate(0);-moz-transform:rotate(0);-o-transform:rotate(0);-ms-transform:rotate(0)}25%{transform:rotate(90deg);-webkit-transform:rotate(90deg);-khtml-transform:rotate(90deg);-moz-transform:rotate(90deg);-o-transform:rotate(90deg);-ms-transform:rotate(90deg)}50%{transform:rotate(180deg);-webkit-transform:rotate(180deg);-khtml-transform:rotate(180deg);-moz-transform:rotate(180deg);-o-transform:rotate(180deg);-ms-transform:rotate(180deg)}75%{transform:rotate(270deg);-webkit-transform:rotate(270deg);-khtml-transform:rotate(270deg);-moz-transform:rotate(270deg);-o-transform:rotate(270deg);-ms-transform:rotate(270deg)}100%{transform:rotate(360deg);-webkit-transform:rotate(360deg);-khtml-transform:rotate(360deg);-moz-transform:rotate(360deg);-o-transform:rotate(360deg);-ms-transform:rotate(360deg)}}@-o-keyframes rotate-effect{0%{transform:rotate(0);-webkit-transform:rotate(0);-khtml-transform:rotate(0);-moz-transform:rotate(0);-o-transform:rotate(0);-ms-transform:rotate(0)}25%{transform:rotate(90deg);-webkit-transform:rotate(90deg);-khtml-transform:rotate(90deg);-moz-transform:rotate(90deg);-o-transform:rotate(90deg);-ms-transform:rotate(90deg)}50%{transform:rotate(180deg);-webkit-transform:rotate(180deg);-khtml-transform:rotate(180deg);-moz-transform:rotate(180deg);-o-transform:rotate(180deg);-ms-transform:rotate(180deg)}75%{transform:rotate(270deg);-webkit-transform:rotate(270deg);-khtml-transform:rotate(270deg);-moz-transform:rotate(270deg);-o-transform:rotate(270deg);-ms-transform:rotate(270deg)}100%{transform:rotate(360deg);-webkit-transform:rotate(360deg);-khtml-transform:rotate(360deg);-moz-transform:rotate(360deg);-o-transform:rotate(360deg);-ms-transform:rotate(360deg)}}@keyframes rotate-effect{0%{transform:rotate(0);-webkit-transform:rotate(0);-khtml-transform:rotate(0);-moz-transform:rotate(0);-o-transform:rotate(0);-ms-transform:rotate(0)}25%{transform:rotate(90deg);-webkit-transform:rotate(90deg);-khtml-transform:rotate(90deg);-moz-transform:rotate(90deg);-o-transform:rotate(90deg);-ms-transform:rotate(90deg)}50%{transform:rotate(180deg);-webkit-transform:rotate(180deg);-khtml-transform:rotate(180deg);-moz-transform:rotate(180deg);-o-transform:rotate(180deg);-ms-transform:rotate(180deg)}75%{transform:rotate(270deg);-webkit-transform:rotate(270deg);-khtml-transform:rotate(270deg);-moz-transform:rotate(270deg);-o-transform:rotate(270deg);-ms-transform:rotate(270deg)}100%{transform:rotate(360deg);-webkit-transform:rotate(360deg);-khtml-transform:rotate(360deg);-moz-transform:rotate(360deg);-o-transform:rotate(360deg);-ms-transform:rotate(360deg)}}.menu-phone li ul.list-v{right:calc(100% - .5 * 16px)}.menu-phone li ul.list-v ul{right:calc(100% - .5 * 16px)}#wrapper{max-width:1080px;margin:auto}@media screen and (min-width:2048px){#wrapper{max-width:55vw}}#wrapper .menu{-webkit-box-flex:1;-moz-box-flex:1;-webkit-flex:1 1;-ms-flex:1 1;flex:1 1;margin:0 16px 0 0}#wrapper .menu .list-v ul{left:calc(100% - .5 * 16px)}.menu-phone{display:-webkit-box;display:-moz-box;display:none;margin-top:16px;right:8px;transition:all .28s ease;-webkit-transition:all .28s ease;-khtml-transition:all 0.28s ease;-moz-transition:all .28s ease;-o-transition:all .28s ease;-ms-transition:all .28s ease}.menu-phone ul{right:calc(100% - .5 * 16px)}@media screen and (max-width:500px){.menu-phone{display:-webkit-box;display:-moz-box;display:block}}.l_header{max-width:65vw;left:calc((100% - 65vw) * .5);border-bottom-left-radius:8px;border-bottom-right-radius:8px}@media screen and (max-width:2048px){.l_header{max-width:1112px;left:calc((100% - 1112px) * .5)}}@media screen and (max-width:1112px){.l_header{left:0;border-radius:0;-webkit-border-radius:0;max-width:100%}}@media screen and (max-width:500px){.l_header .container{margin-left:0;margin-right:0}.l_header #wrapper .nav-main .title{padding-left:16px;padding-right:16px}.l_header #wrapper .nav-sub{width:100%}.l_header #wrapper .nav-sub .title{overflow-y:scroll;margin-top:2px;padding:8px 16px}.l_header #wrapper .switcher{display:-webkit-box;display:-moz-box;display:-ms-flexbox;display:-webkit-flex;display:flex;display:flex;margin-right:8px}.l_header .menu{display:-webkit-box;display:-moz-box;display:none}}@media screen and (max-width:500px){.list-v li{max-width:270px}}#u-search{display:-webkit-box;display:-moz-box;display:none;position:fixed;top:0;left:0;width:100%;height:100%;padding:60px 20px;z-index:1001}@media screen and (max-width:680px){#u-search{padding:0}}@media screen and (prefers-color-scheme:dark) and (max-width:500px){.l_header .m_search{background:var(--color-site-bg)!important}}.cover-wrapper .cover-body .subtitle{font-size:.875rem}.fa-brands.color-github,.fa-duotone.color-github,.fa-light.color-github,.fa-regular.color-github,.fa-solid.color-github,.fa-thin.color-github,.fa.color-github,.fad.color-github,.fal.color-github,.far.color-github,.fas.color-github,.iziToast>.iziToast-body .iziToast-icon.color-github{color:#000}.fa-brands.color-friends,.fa-duotone.color-friends,.fa-light.color-friends,.fa-regular.color-friends,.fa-solid.color-friends,.fa-thin.color-friends,.fa.color-friends,.fad.color-friends,.fal.color-friends,.far.color-friends,.fas.color-friends,.iziToast>.iziToast-body .iziToast-icon.color-friends{color:#d31ee9}.fa-brands.color-about,.fa-duotone.color-about,.fa-light.color-about,.fa-regular.color-about,.fa-solid.color-about,.fa-thin.color-about,.fa.color-about,.fad.color-about,.fal.color-about,.far.color-about,.fas.color-about,.iziToast>.iziToast-body .iziToast-icon.color-about{color:#0095ff}.fa-brands.color-travellings,.fa-duotone.color-travellings,.fa-light.color-travellings,.fa-regular.color-travellings,.fa-solid.color-travellings,.fa-thin.color-travellings,.fa.color-travellings,.fad.color-travellings,.fal.color-travellings,.far.color-travellings,.fas.color-travellings,.iziToast>.iziToast-body .iziToast-icon.color-travellings{color:#734ae6}</style><link rel="stylesheet" href="/css/style.css?v=2d24f6fe3e" media="print" onload="this.media="all",this.onload=null"><noscript><link rel="stylesheet" href="/css/style.css?v=2d24f6fe3e"></noscript><script>let userColorScheme=localStorage.getItem("color-scheme");userColorScheme&&document.documentElement.setAttribute("color-scheme",userColorScheme)</script><script>window.MSInputMethodContext&&document.documentMode&&document.write('<style>html{overflow-x: hidden !important;overflow-y: hidden !important;}.kill-ie{text-align:center;height: 100%;margin-top: 15%;margin-bottom: 5500%;}.kill-t{font-size: 2rem;}.kill-c{font-size: 1.2rem;}#l_header,#l_body{display: none;}</style><div class="kill-ie"><span class="kill-t"><b>抱歉，您的浏览器无法访问本站</b></span><br/><span class="kill-c">微软已经于2016年终止了对 Internet Explorer (IE) 10 及更早版本的支持，<br/>继续使用存在极大的安全隐患，请使用当代主流的浏览器进行访问。</span><br/><a target="_blank" rel="noopener" href="https://blogs.windows.com/windowsexperience/2021/05/19/the-future-of-internet-explorer-on-windows-10-is-in-microsoft-edge/"><strong>了解详情 ></strong></a></div>')</script><noscript><style>html{overflow-x:hidden!important;overflow-y:hidden!important}.kill-noscript{text-align:center;height:100%;margin-top:15%;margin-bottom:5500%}.kill-t{font-size:2rem}.kill-c{font-size:1.2rem}#l_body,#l_header{display:none}</style><div class="kill-noscript"> <span class="kill-t"><b>抱歉，您的浏览器无法访问本站</b></span><br> <span class="kill-c">本页面需要浏览器支持（启用）JavaScript</span><br> <a target="_blank" rel="external nofollow noopener noreferrer" href="https://www.baidu.com/s?wd=启用JavaScript"><strong>了解详情 ></strong></a></div></noscript><script>function volantisEventListener(e,t,o){this.type=e,this.f=t,this.ele=o}function volantisDom(e){return e||(e=document.createElement("div")),this.ele=e,this.ele.find=e=>{let t=this.ele.querySelector(e);if(t)return new volantisDom(t)},this.ele.hasClass=e=>this.ele.className.match(new RegExp("(\\s|^)"+e+"(\\s|$)")),this.ele.addClass=e=>(this.ele.classList.add(e),this.ele),this.ele.removeClass=e=>(this.ele.classList.remove(e),this.ele),this.ele.toggleClass=e=>(this.ele.hasClass(e)?this.ele.removeClass(e):this.ele.addClass(e),this.ele),this.ele.on=(e,t,o=1)=>(this.ele.addEventListener(e,t,!1),o&&volantis.EventListener.list.push(new volantisEventListener(e,t,this.ele)),this.ele),this.ele.click=(e,t)=>(this.ele.on("click",e,t),this.ele),this.ele.scroll=(e,t)=>(this.ele.on("scroll",e,t),this.ele),this.ele.html=e=>(this.ele.innerHTML=e,this.ele),this.ele.hide=e=>(this.ele.style.display="none",this.ele),this.ele.show=e=>(this.ele.style.display="block",this.ele),this.ele}function RunItem(){function e(e,t){this.name=t||e.name,this.run=()=>{try{e()}catch(e){console.log(e)}}}this.list=[],this.start=()=>{for(var e=0;e<this.list.length;e++)this.list[e].run()},this.push=(t,o,n=!0)=>{let l=t;n&&(l=()=>{volantis.requestAnimationFrame(t)});var s=new e(l,o);this.list.push(s)},this.remove=e=>{for(let t=0;t<this.list.length;t++){this.list[t].name==e&&this.list.splice(t,1)}}}function errorImgAvatar(e){e.src="https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/avatar/round/3442075.svg",e.onerror=null}function errorImgCover(e){e.src="https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/cover/76b86c0226ffd.svg",e.onerror=null}window.volantis={},volantis.debug="false",volantis.dom={},volantis.EventListener={},volantis.EventListener.list=[],volantis.EventListener.remove=()=>{volantis.EventListener.list.forEach((function(e){e.ele.removeEventListener(e.type,e.f,!1)})),volantis.EventListener.list=[]},volantis.dom.$=e=>e?new volantisDom(e):null,volantis.pjax={},volantis.pjax.method={complete:new RunItem,error:new RunItem,send:new RunItem},volantis.pjax=Object.assign(volantis.pjax,{push:volantis.pjax.method.complete.push,error:volantis.pjax.method.error.push,send:volantis.pjax.method.send.push}),volantis.rightmenu={},volantis.rightmenu.method={handle:new RunItem},volantis.rightmenu=Object.assign(volantis.rightmenu,{handle:volantis.rightmenu.method.handle.push}),volantis.dark={},volantis.dark.method={toggle:new RunItem},volantis.dark=Object.assign(volantis.dark,{push:volantis.dark.method.toggle.push}),volantis.js=(e,t)=>new Promise((o=>{setTimeout((function(){var n=document.getElementsByTagName("head")[0]||document.documentElement,l=document.createElement("script");if(l.setAttribute("type","text/javascript"),t)if(JSON.stringify(t))for(let e in t)"onload"==e?l[e]=()=>{t[e](),o()}:(l[e]=t[e],l.onload=o);else l.onload=()=>{t(),o()};else l.onload=o;l.setAttribute("src",e),n.appendChild(l)}))})),volantis.css=e=>new Promise((t=>{setTimeout((function(){var o=document.createElement("link");o.rel="stylesheet",o.href=e,o.onload=t,document.getElementsByTagName("head")[0].appendChild(o)}))})),volantis.import={jQuery:()=>"undefined"==typeof jQuery?volantis.js("https://cdn.bootcdn.net/ajax/libs/jquery/3.6.0/jquery.min.js"):new Promise((e=>{e()}))},volantis.requestAnimationFrame=e=>{window.requestAnimationFrame||(window.requestAnimationFrame=window.requestAnimationFrame||window.mozRequestAnimationFrame||window.webkitRequestAnimationFrame),window.requestAnimationFrame(e)},volantis.layoutHelper=(e,t,o)=>{function n(e,t,o){volantis.tempDiv=document.createElement("div"),volantis.tempDiv.innerHTML=t;let n=document.querySelector("#layoutHelper-"+e);n&&(o&&(n.innerHTML=""),n.append(volantis.tempDiv))}o=Object.assign({clean:!1,pjax:!0},o),n(e,t,o.clean),o.pjax&&volantis.pjax.push((()=>{n(e,t,o.clean)}),"layoutHelper-"+e)},volantis.scroll={engine:new RunItem,unengine:new RunItem},volantis.scroll=Object.assign(volantis.scroll,{push:volantis.scroll.engine.push}),volantis.scroll.getScrollTop=()=>{let e;return window.pageYOffset?e=window.pageYOffset:document.compatMode&&"BackCompat"!=document.compatMode?e=document.documentElement.scrollTop:document.body&&(e=document.body.scrollTop),e},volantis.scroll.scrollHeight=function(){return Math.max(document.body.scrollHeight,document.documentElement.scrollHeight)},volantis.scroll.offsetHeight=function(){return Math.max(document.body.offsetHeight,document.documentElement.offsetHeight,document.body.clientHeight,document.documentElement.clientHeight)},volantis.scroll.progress=function(){return volantis.scroll.getScrollTop()/(volantis.scroll.scrollHeight()-volantis.scroll.offsetHeight())},volantis.scroll.handleScrollEvents=()=>{volantis.scroll.lastScrollTop=volantis.scroll.getScrollTop(),volantis.requestAnimationFrame((function e(){const t=volantis.scroll.getScrollTop();volantis.scroll.lastScrollTop!==t?(volantis.scroll.del=t-volantis.scroll.lastScrollTop,volantis.scroll.lastScrollTop=t,volantis.scroll.unengine.list=[],volantis.scroll.engine.start()):volantis.scroll.unengine.start(),volantis.requestAnimationFrame(e)}))},volantis.scroll.handleScrollEvents(),volantis.scroll.ele=null,volantis.scroll.to=(e,t={})=>{e&&(volantis.scroll.ele=e,opt={top:e.getBoundingClientRect().top+document.documentElement.scrollTop,behavior:"smooth"},"top"in t&&(opt.top=t.top),"behavior"in t&&(opt.behavior=t.behavior),"addTop"in t&&(opt.top+=t.addTop),"observerDic"in t||(t.observerDic=100),window.scrollTo(opt),t.observer&&setTimeout((()=>{volantis.scroll.ele==e&&volantis.scroll.unengine.push((()=>{let o=e.getBoundingClientRect().top;o>=-t.observerDic&&o<=t.observerDic||volantis.scroll.to(e,t),volantis.scroll.unengine.remove("unengineObserver")}),"unengineObserver")}),1e3))},volantis.cleanContentVisibility=()=>{document.querySelector(".post-story")&&(console.log("cleanContentVisibility"),document.querySelectorAll(".post-story").forEach((e=>{e.classList.remove("post-story")})))}</script><script>volantis.GLOBAL_CONFIG={root:"/",debug:!1,default:{avatar:"https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/avatar/round/3442075.svg",link:"https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/link/8f277b4ee0ecd.svg",cover:"https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/cover/76b86c0226ffd.svg",image:"https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/media/placeholder/image/2659360.svg"},lastupdate:new Date(1739832285375),cdn:{izitoast_css:"https://cdn.bootcdn.net/ajax/libs/izitoast/1.4.0/css/iziToast.min.css",izitoast_js:"https://cdn.bootcdn.net/ajax/libs/izitoast/1.4.0/js/iziToast.min.js",fancybox_css:"https://cdn.bootcdn.net/ajax/libs/fancyapps-ui/4.0.31/fancybox.min.css",fancybox_js:"https://cdn.bootcdn.net/ajax/libs/fancyapps-ui/4.0.31/fancybox.umd.min.js"},sidebar:{for_page:["blogger","navigation","webinfo"],for_post:["toc"],webinfo:{lastupd:{enable:!1,friendlyShow:!0},runtime:{data:"2019/08/19",unit:"天"}}},plugins:{message:{enable:!1,icon:{default:"fa-solid fa-info-circle light-blue",quection:"fa-solid fa-question-circle light-blue"},time:{default:5e3,quection:2e4},position:"topRight",transitionIn:"bounceInLeft",transitionOut:"fadeOutRight",titleColor:"var(--color-text)",messageColor:"var(--color-text)",backgroundColor:"var(--color-card)",zindex:2147483647,copyright:{enable:!0,title:"知识共享许可协议",message:"请遵守 CC BY-NC-SA 4.0 协议。",icon:"far fa-copyright light-blue"},aplayer:{enable:!0,play:"fa-solid fa-play",pause:"fa-solid fa-pause"},rightmenu:{enable:!0,notice:!0}},aplayer:{id:1480000098,enable:!0}},search:{dataPath:("/".endsWith("/")?"/":"//")+"content.json"},languages:{search:{hits_empty:"找不到您查询的内容：${query}",hits_stats:"找到 ${hits} 条结果，用时 ${time} 毫秒"}}}</script></head><body itemscope itemtype="http://schema.org/WebPage"><header itemscope itemtype="http://schema.org/WPHeader" id="l_header" class="l_header auto shadow floatable blur show" style="opacity:0"><div class="container"><div id="wrapper"><div class="nav-sub"><p class="title"></p><ul class="switcher nav-list-h m-phone" id="pjax-header-nav-list"><li><a id="s-comment" class="fa-solid fa-comments fa-fw" target="_self" title="comment"></a></li><li><a id="s-toc" class="s-toc fa-solid fa-list fa-fw" target="_self" title="toc"></a></li></ul></div><div class="nav-main"> <a class="title flat-box" target="_self" href="/">MHuiG's Blog</a><div class="menu navigation"><ul class="nav-list-h m-pc"><li><a class="menuitem flat-box faa-parent animated-hover" title="索引"><i class="fa-duotone fa-list-alt fa-fw"></i> 索引</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="/notes/" title="便签" active-action="action-notes"><i class="fa-duotone fa-books fa-fw"></i> 便签</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/categories/" title="分类" active-action="action-categories"><i class="fa-duotone fa-folder-open fa-fw"></i> 分类</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/tags/" title="标签" active-action="action-tags"><i class="fa-duotone fa-tags fa-fw"></i> 标签</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/archives/" title="归档" active-action="action-archives"><i class="fa-duotone fa-archive fa-fw"></i> 归档</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/friends/" title="友链" active-action="action-pagesfriends"><i class="fa-duotone fa-link fa-fw"></i> 友链</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="https://www.travellings.cn/go-by-clouds.html" title="Travelling" target="_blank" active-action="action-https:wwwtravellingscngo-by-cloudshtml" rel="external nofollow noopener noreferrer"><i class="fa-duotone fa-subway fa-fw"></i> Travelling</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/about/" title="关于" active-action="action-pagesabout"><i class="fa-duotone fa-user-tie fa-fw"></i> 关于</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="noopener" href="https://mhuig.top/privacy-policy/" title="隐私政策" active-action="action-https:mhuigtopprivacy-policy"><i class="fa-duotone fa-user-secret fa-fw"></i> 隐私政策</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" title="更多"><i class="fa-duotone fa-fan fa-spin fa-fw"></i> 更多</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/beer/" title="Sponsor" active-action="action-pagesbeer"><i class="fa-duotone fa-heart-square fa-fw"></i> Sponsor</a></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" title="博客管理"><i class="fa-duotone fa-user-shield fa-fw"></i> 博客管理</a><ul class="list-v"><li><a class="menuitem flat-box header toggle-mode-btn"><i class="fa-duotone fa-bat fa-fw"></i> 暗黑模式</a></li><li></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/rss/" title="RSS订阅" active-action="action-pagesrss"><i class="fa-duotone fa-rss fa-fw"></i> RSS订阅</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/sitemap.xml" title="站点地图" active-action="action-sitemapxml"><i class="fa-duotone fa-sitemap fa-fw"></i> 站点地图</a></li><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://mhuig.instatus.com/" title="Monitors" active-action="action-https:mhuiginstatuscom"><i class="fa-duotone fa-telescope fa-fw"></i> Monitors</a></li><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://ssl.mhuig.top/" title="SSL Status" active-action="action-https:sslmhuigtop"><i class="fa-brands fa-expeditedssl fa-fw"></i> SSL Status</a></li></ul></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" title="工具箱"><i class="fa-duotone fa-tools fa-fw"></i> 工具箱</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://rssbox.mhuig.top/" title="RSS Box" active-action="action-https:rssboxmhuigtop"><i class="fa-duotone fa-conveyor-belt-boxes fa-fw"></i> RSS Box</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/havefun/Coco/" title="My Cat" active-action="action-havefunCoco"><i class="fa-duotone fa-cat fa-fw"></i> My Cat</a></li><li><a class="menuitem flat-box"><i class="fa-duotone fa-compact-disc fa-spin fa-fw music"></i> Music</a><ul class="list-v"><li><div class="aplayer-container"><div class="aplayer-local"></div></div></li></ul></li><li></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/talk/" title="碎言碎语" active-action="action-pagestalk"><i class="fa-duotone fa-comments fa-fw"></i> 碎言碎语</a></li></ul></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/MHuiG" title="GitHub" active-action="action-https:githubcomMHuiG"><i class="fa-brands fa-github fa-fw"></i> GitHub</a></li></ul></li></ul></div><div class="m_search"><form name="searchform" class="form u-search-form"><i class="icon fa-solid fa-search fa-fw"></i> <input type="text" class="input u-search-input" placeholder="Search..."></form></div><ul class="switcher nav-list-h m-phone"><li><a class="s-search fa-solid fa-search fa-fw" target="_self" title="search"></a></li><li><a class="s-menu fa-solid fa-bars fa-fw" target="_self" title="menu"></a><ul class="menu-phone list-v navigation white-box"><li><a class="menuitem flat-box faa-parent animated-hover" title="索引"><i class="fa-duotone fa-list-alt fa-fw"></i> 索引</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="/notes/" title="便签" active-action="action-notes"><i class="fa-duotone fa-books fa-fw"></i> 便签</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/categories/" title="分类" active-action="action-categories"><i class="fa-duotone fa-folder-open fa-fw"></i> 分类</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/tags/" title="标签" active-action="action-tags"><i class="fa-duotone fa-tags fa-fw"></i> 标签</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/archives/" title="归档" active-action="action-archives"><i class="fa-duotone fa-archive fa-fw"></i> 归档</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/friends/" title="友链" active-action="action-pagesfriends"><i class="fa-duotone fa-link fa-fw"></i> 友链</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="https://www.travellings.cn/go-by-clouds.html" title="Travelling" target="_blank" active-action="action-https:wwwtravellingscngo-by-cloudshtml" rel="external nofollow noopener noreferrer"><i class="fa-duotone fa-subway fa-fw"></i> Travelling</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/about/" title="关于" active-action="action-pagesabout"><i class="fa-duotone fa-user-tie fa-fw"></i> 关于</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="noopener" href="https://mhuig.top/privacy-policy/" title="隐私政策" active-action="action-https:mhuigtopprivacy-policy"><i class="fa-duotone fa-user-secret fa-fw"></i> 隐私政策</a></li></ul></li><li><a class="menuitem flat-box faa-parent animated-hover" title="更多"><i class="fa-duotone fa-fan fa-spin fa-fw"></i> 更多</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/beer/" title="Sponsor" active-action="action-pagesbeer"><i class="fa-duotone fa-heart-square fa-fw"></i> Sponsor</a></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" title="博客管理"><i class="fa-duotone fa-user-shield fa-fw"></i> 博客管理</a><ul class="list-v"><li><a class="menuitem flat-box header toggle-mode-btn"><i class="fa-duotone fa-bat fa-fw"></i> 暗黑模式</a></li><li></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/rss/" title="RSS订阅" active-action="action-pagesrss"><i class="fa-duotone fa-rss fa-fw"></i> RSS订阅</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/sitemap.xml" title="站点地图" active-action="action-sitemapxml"><i class="fa-duotone fa-sitemap fa-fw"></i> 站点地图</a></li><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://mhuig.instatus.com/" title="Monitors" active-action="action-https:mhuiginstatuscom"><i class="fa-duotone fa-telescope fa-fw"></i> Monitors</a></li><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://ssl.mhuig.top/" title="SSL Status" active-action="action-https:sslmhuigtop"><i class="fa-brands fa-expeditedssl fa-fw"></i> SSL Status</a></li></ul></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" title="工具箱"><i class="fa-duotone fa-tools fa-fw"></i> 工具箱</a><ul class="list-v"><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://rssbox.mhuig.top/" title="RSS Box" active-action="action-https:rssboxmhuigtop"><i class="fa-duotone fa-conveyor-belt-boxes fa-fw"></i> RSS Box</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/havefun/Coco/" title="My Cat" active-action="action-havefunCoco"><i class="fa-duotone fa-cat fa-fw"></i> My Cat</a></li><li><a class="menuitem flat-box faa-parent animated-hover" href="/pages/talk/" title="碎言碎语" active-action="action-pagestalk"><i class="fa-duotone fa-comments fa-fw"></i> 碎言碎语</a></li></ul></li><hr><li><a class="menuitem flat-box faa-parent animated-hover" target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/MHuiG" title="GitHub" active-action="action-https:githubcomMHuiG"><i class="fa-brands fa-github fa-fw"></i> GitHub</a></li></ul></li></ul></li></ul></div></div></div></header><div id="l_body"><div id="l_cover"><div id="none" class="cover-wrapper docs featured" style="display:none"><div class="cover-bg lazyload placeholder" data-bg></div><div class="cover-body"><div class="top"><p class="title">MHuiG</p><p class="subtitle">「看庭前花开花落,望天上云卷云舒」</p></div><div class="bottom"><div class="menu navigation"><div class="list-h"><a target="_blank" rel="external nofollow noopener noreferrer" href="https://github.com/MHuiG" active-action="action-https:githubcomMHuiG"><i class="fa-brands fa-github color-github fa-fw"></i><p>Github</p></a><a href="/pages/friends/" active-action="action-pagesfriends"><i class="fa-duotone fa-link color-friends fa-fw"></i><p>友链</p></a><a href="/pages/about/" active-action="action-pagesabout"><i class="fa-duotone fa-user-tie color-about fa-fw"></i><p>关于</p></a><a href="https://www.travellings.cn/go-by-clouds.html" target="_blank" active-action="action-https:wwwtravellingscngo-by-cloudshtml" rel="external nofollow noopener noreferrer"><i class="fa-duotone fa-subway color-travellings fa-fw"></i><p>Travelling</p></a></div></div></div></div><div id="scroll-down" style="display:none"><i class="fa fa-chevron-down scroll-down-effects"></i></div></div></div><div id="safearea"><div class="body-wrapper"><div id="l_main" class><article itemscope itemtype="http://schema.org/Article" class="article post white-box reveal md shadow floatable blur article-type-docs" id="docs" itemprop="blogPost"><link itemprop="mainEntityOfPage" href="https://blog.mhuig.top/notes/Spark/k-means"><span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization"><meta itemprop="name" content="MHuiG"></span><span hidden itemprop="post" itemscope itemtype="http://schema.org/Post"><meta itemprop="name" content="MHuiG"><meta itemprop="description" content="MHuiG&#39;s Blog (MHuiG的博客) MHuiG&#39;s Neverland（MHuiG的梦幻岛） —— MHuiG(@MHuiG) 随便写写画画的地方 - 技术博客"></span><span hidden><meta itemprop="image" content="/lib/favicon/android-chrome-192x192.png"></span><div class="article-meta" id="top"><span hidden itemprop="name headline"></span></div><div id="layoutHelper-page-plugins"></div><div id="post-body" itemprop="articleBody"><p> <span class="p logo center large">K-Means</span></p><br><h1 hidden>基于 K 均值聚类的网络流量异常检测 (pyspark)</h1><p>异常检测常用于检测欺诈、网络攻击、服务器及传感设备故障。在这些应用中，我们要能够找出以前从未见过的新型异常，如新欺诈方式、新入侵方法或新服务器故障模式。</p><div class="story post-story"><h2 id="数据集"><a href="#数据集" class="headerlink" title="数据集"></a>数据集</h2><p>KDD Cup 1999 数据集</p><span class="btn center large"><a class="button" target="_blank" rel="external nofollow noopener noreferrer" href="http://kdd.ics.uci.edu/databases/kddcup99/kddcup99.html" title="下载 KDD Cup 1999 数据集"><i class="fas fa-download"></i> 下载 KDD Cup 1999 数据集</a></span><p>数据集为 CSV 格式，每个连接占一行，包含 38 个特征。</p><p>单行示例：</p><figure class="highlight properties"><table><tbody><tr><td class="code"><pre><span class="line"><span class="attr">0,tcp,http,SF,239,486,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,8,8,0.00,0.00,0.00,0.00,1.00,0.00,0.00,19,19,1.00,0.00,0.05,0.00,0.00,0.00,0.00,0.00,normal.</span></span><br></pre></td></tr></tbody></table></figure><p>最后的字段表示类别标号。大多数标号为 normal.， 但也有一些样本代表各种网络攻击。</p></div><div class="story post-story"><h2 id="算法"><a href="#算法" class="headerlink" title="算法"></a>算法</h2><h3 id="K均值聚类算法"><a href="#K均值聚类算法" class="headerlink" title="K均值聚类算法"></a><strong>K 均值聚类算法</strong></h3><p>聚类算法是指将一堆没有标签的数据自动划分成几类的方法，这个方法要保证同一类的数据有相似的特征。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651998573990/20208216374.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651998573990/20208216374.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="K均值聚类算法"></p><h3 id="算法过程"><a href="#算法过程" class="headerlink" title="算法过程"></a><strong>算法过程</strong></h3><p>K-Means 算法的特点是类别的个数是人为给定的。是一个迭代求解的聚类算法，属于划分型的聚类方法，即首先创建 K 个划分，然后迭代地将样本从一个划分转移到另一个划分来改善最终聚类的效果。其过程大致如下。</p><p>（1）根据给定的 K 值选取 K 个样本点作为初始划分中心。</p><p>（2）计算所有样本点到每一个划分中心的距离，并将所有样本点划分到距离最近的划分中心。</p><p>（3）计算每个划分中样本点的平均值，并将其作为新的中心。</p><p>（4）循环进行步骤（2）和步骤（3）直至最大迭代次数，或划分中心的变化小于某一预定义阈值。</p><h3 id="伪代码"><a href="#伪代码" class="headerlink" title="伪代码"></a><strong>伪代码</strong></h3><figure class="highlight matlab"><table><tbody><tr><td class="code"><pre><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">K</span>-<span class="title">Means</span><span class="params">(输入数据，中心点个数K)</span> </span></span><br><span class="line"> 获取输入数据的维度Dim和个数N </span><br><span class="line"> 随机生成K个Dim维的点 </span><br><span class="line"> <span class="keyword">while</span>(算法未收敛) </span><br><span class="line"> 对N个点：计算每个点属于哪一类。 </span><br><span class="line"> 对于K个中心点： </span><br><span class="line"> <span class="number">1</span>，找出所有属于自己这一类的所有数据点 </span><br><span class="line"> <span class="number">2</span>，把自己的坐标修改为这些数据点的中心点坐标 </span><br><span class="line"> <span class="keyword">end</span></span><br><span class="line"> 输出结果</span><br><span class="line"><span class="keyword">end</span></span><br></pre></td></tr></tbody></table></figure><p>K-Means 的一个重要的假设是：数据之间的相似度可以使用欧氏距离度量，如果不能使用欧氏距离度量，要先把数据转换到能用欧氏距离度量，这一点很重要。可以使用欧氏距离度量的意思就是欧氏距离越小，两个数据相似度越高。</p><p>假设簇划分为（<mjx-container class="MathJax" jax="SVG"><svg style="vertical-align:-.339ex" xmlns="http://www.w3.org/2000/svg" width="2.605ex" height="1.934ex" role="img" focusable="false" viewBox="0 -705 1151.6 855"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(1,-1)"><g data-mml-node="math"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D436" d="M50 252Q50 367 117 473T286 641 490 704Q580 704 633 653 642 643 648 636T656 626L657 623Q660 623 684 649 691 655 699 663T715 679 725 690L740 705H746Q760 705 760 698 760 694 728 561 692 422 692 421 690 416 687 415T669 413H653Q647 419 647 422 647 423 648 429T650 449 651 481Q651 552 619 605T510 659Q484 659 454 652T382 628 299 572 226 479Q194 422 175 346T156 222Q156 108 232 58 280 24 350 24 441 24 512 92T606 240Q610 253 612 255T628 257Q648 257 648 248 648 243 647 239 618 132 523 55T319-22Q206-22 128 53T50 252Z"/></g><g data-mml-node="mn" transform="translate(748,-150) scale(0.707)"><path data-c="31" d="M213 578 200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641 273 663Q275 666 285 666 294 666 302 660V361L303 61Q310 54 315 52T339 48 401 46H427V0H416Q395 3 257 3 121 3 100 0H88V46H114Q136 46 152 46T177 47 193 50 201 52 207 57 213 61V578Z"/></g></g></g></g></svg></mjx-container>,<mjx-container class="MathJax" jax="SVG"><svg style="vertical-align:-.339ex" xmlns="http://www.w3.org/2000/svg" width="2.605ex" height="1.934ex" role="img" focusable="false" viewBox="0 -705 1151.6 855"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(1,-1)"><g data-mml-node="math"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D436" d="M50 252Q50 367 117 473T286 641 490 704Q580 704 633 653 642 643 648 636T656 626L657 623Q660 623 684 649 691 655 699 663T715 679 725 690L740 705H746Q760 705 760 698 760 694 728 561 692 422 692 421 690 416 687 415T669 413H653Q647 419 647 422 647 423 648 429T650 449 651 481Q651 552 619 605T510 659Q484 659 454 652T382 628 299 572 226 479Q194 422 175 346T156 222Q156 108 232 58 280 24 350 24 441 24 512 92T606 240Q610 253 612 255T628 257Q648 257 648 248 648 243 647 239 618 132 523 55T319-22Q206-22 128 53T50 252Z"/></g><g data-mml-node="mn" transform="translate(748,-150) scale(0.707)"><path data-c="32" d="M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315 301 241Q265 210 201 149L142 93 218 92Q375 92 385 97 392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19 31Q50 38 56 46T86 81Q115 113 136 137 145 147 170 174T204 211 233 244 261 278 284 308 305 340 320 369 333 401 340 431 343 464Q343 527 309 573T212 619Q179 619 154 602T119 569 109 550Q109 549 114 549 132 549 151 535T170 489Q170 464 154 447T109 429Z"/></g></g></g></g></svg></mjx-container>,…<mjx-container class="MathJax" jax="SVG"><svg style="vertical-align:-.357ex" xmlns="http://www.w3.org/2000/svg" width="2.639ex" height="1.952ex" role="img" focusable="false" viewBox="0 -705 1166.4 862.8"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(1,-1)"><g data-mml-node="math"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D436" d="M50 252Q50 367 117 473T286 641 490 704Q580 704 633 653 642 643 648 636T656 626L657 623Q660 623 684 649 691 655 699 663T715 679 725 690L740 705H746Q760 705 760 698 760 694 728 561 692 422 692 421 690 416 687 415T669 413H653Q647 419 647 422 647 423 648 429T650 449 651 481Q651 552 619 605T510 659Q484 659 454 652T382 628 299 572 226 479Q194 422 175 346T156 222Q156 108 232 58 280 24 350 24 441 24 512 92T606 240Q610 253 612 255T628 257Q648 257 648 248 648 243 647 239 618 132 523 55T319-22Q206-22 128 53T50 252Z"/></g><g data-mml-node="mi" transform="translate(748,-150) scale(0.707)"><path data-c="1D458" d="M121 647Q121 657 125 670T137 683Q138 683 209 688T282 694Q294 694 294 686 294 679 244 477 194 279 194 272 213 282 223 291 247 309 292 354T362 415Q402 442 438 442 468 442 485 423T503 369Q503 344 496 327T477 302 456 291 438 288Q418 288 406 299T394 328Q394 353 410 369T442 390L458 393Q446 405 434 405H430Q398 402 367 380T294 316 228 255Q230 254 243 252T267 246 293 238 320 224 342 206 359 180 365 147Q365 130 360 106T354 66Q354 26 381 26 429 26 459 145 461 153 479 153H483Q499 153 499 144 499 139 496 130 455-11 378-11 333-11 305 15T277 90Q277 108 280 121T283 145Q283 167 269 183T234 206 200 217 182 220H180Q168 178 159 139T145 81 136 44 129 20 122 7 111-2Q98-11 83-11 66-11 57-1T48 16Q48 26 85 176T158 471L195 616Q196 629 188 632T149 637H144Q134 637 131 637T124 640 121 647Z"/></g></g></g></g></svg></mjx-container>）, 则优化目标是最小化平方误差 SSE:</p><p><mjx-container class="MathJax" jax="SVG" display="true" width="full" style="min-width:34.081ex"><svg style="vertical-align:-2.905ex;min-width:34.081ex" xmlns="http://www.w3.org/2000/svg" width="100%" height="6.941ex" role="img" focusable="false"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(0.0181,-0.0181) translate(0, -1783.9)"><g data-mml-node="math"><g data-mml-node="mtable" transform="translate(2078,0) translate(-2078,0)"><g transform="translate(0 1783.9) matrix(1 0 0 -1 0 0) scale(55.25)"><svg data-table="true" preserveAspectRatio="xMidYMid" viewBox="5453.9 -1783.9 1 3067.8"><g transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mlabeledtr" transform="translate(0,43.2)"><g data-mml-node="mtd"><g data-mml-node="mi"><path data-c="1D446" d="M308 24Q367 24 416 76T466 197Q466 260 414 284 308 311 278 321T236 341Q176 383 176 462 176 523 208 573T273 648Q302 673 343 688T407 704H418 425Q521 704 564 640 565 640 577 653T603 682 623 704Q624 704 627 704T632 705Q645 705 645 698T617 577 585 459 569 456Q549 456 549 465 549 471 550 475 550 478 551 494T553 520Q553 554 544 579T526 616 501 641Q465 662 419 662 362 662 313 616T263 510Q263 480 278 458T319 427Q323 425 389 408T456 390Q490 379 522 342T554 242Q554 216 546 186 541 164 528 137T492 78 426 18 332-20Q320-22 298-22 199-22 144 33L134 44 106 13Q83-14 78-18T65-22Q52-22 52-14 52-11 110 221 112 227 130 227H143Q149 221 149 216 149 214 148 207T144 186 142 153Q144 114 160 87T203 47 255 29 308 24Z"/></g><g data-mml-node="mi" transform="translate(645,0)"><path data-c="1D446" d="M308 24Q367 24 416 76T466 197Q466 260 414 284 308 311 278 321T236 341Q176 383 176 462 176 523 208 573T273 648Q302 673 343 688T407 704H418 425Q521 704 564 640 565 640 577 653T603 682 623 704Q624 704 627 704T632 705Q645 705 645 698T617 577 585 459 569 456Q549 456 549 465 549 471 550 475 550 478 551 494T553 520Q553 554 544 579T526 616 501 641Q465 662 419 662 362 662 313 616T263 510Q263 480 278 458T319 427Q323 425 389 408T456 390Q490 379 522 342T554 242Q554 216 546 186 541 164 528 137T492 78 426 18 332-20Q320-22 298-22 199-22 144 33L134 44 106 13Q83-14 78-18T65-22Q52-22 52-14 52-11 110 221 112 227 130 227H143Q149 221 149 216 149 214 148 207T144 186 142 153Q144 114 160 87T203 47 255 29 308 24Z"/></g><g data-mml-node="mi" transform="translate(1290,0)"><path data-c="1D438" d="M492 213Q472 213 472 226 472 230 477 250T482 285Q482 316 461 323T364 330H312Q311 328 277 192T243 52Q243 48 254 48T334 46Q428 46 458 48T518 61Q567 77 599 117T670 248Q680 270 683 272 690 274 698 274 718 274 718 261 613 7 608 2 605 0 322 0H133Q31 0 31 11 31 13 34 25 38 41 42 43T65 46Q92 46 125 49 139 52 144 61 146 66 215 342T285 622Q285 629 281 629 273 632 228 634H197Q191 640 191 642T193 659Q197 676 203 680H757Q764 676 764 669 764 664 751 557T737 447Q735 440 717 440H705Q698 445 698 453L701 476Q704 500 704 528 704 558 697 578T678 609 643 625 596 632 532 634H485Q397 633 392 631 388 629 386 622 385 619 355 499T324 377Q347 376 372 376H398Q464 376 489 391T534 472Q538 488 540 490T557 493Q562 493 565 493T570 492 572 491 574 487 577 483L544 351Q511 218 508 216 505 213 492 213Z"/></g><g data-mml-node="mo" transform="translate(2331.8,0)"><path data-c="3D" d="M56 347Q56 360 70 367H707Q722 359 722 347 722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153 722 140 707 133H70Q56 140 56 153Z"/></g><g data-mml-node="munderover" transform="translate(3387.6,0)"><g data-mml-node="mo"><path data-c="2211" d="M60 948Q63 950 665 950H1267L1325 815Q1384 677 1388 669H1348L1341 683Q1320 724 1285 761 1235 809 1174 838T1033 881 882 898 699 902H574 543 251L259 891Q722 258 724 252 725 250 724 246 721 243 460-56L196-356Q196-357 407-357 459-357 548-357T676-358Q812-358 896-353T1063-332 1204-283 1307-196Q1328-170 1348-124H1388Q1388-125 1381-145T1356-210 1325-294L1267-449 666-450Q64-450 61-448 55-446 55-439 55-437 57-433L590 177Q590 178 557 222T452 366 322 544L56 909 55 924Q55 945 60 948Z"/></g><g data-mml-node="TeXAtom" transform="translate(148.2,-1087.9) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mo" transform="translate(345,0)"><path data-c="3D" d="M56 347Q56 360 70 367H707Q722 359 722 347 722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153 722 140 707 133H70Q56 140 56 153Z"/></g><g data-mml-node="mn" transform="translate(1123,0)"><path data-c="31" d="M213 578 200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641 273 663Q275 666 285 666 294 666 302 660V361L303 61Q310 54 315 52T339 48 401 46H427V0H416Q395 3 257 3 121 3 100 0H88V46H114Q136 46 152 46T177 47 193 50 201 52 207 57 213 61V578Z"/></g></g><g data-mml-node="mi" transform="translate(537.8,1150) scale(0.707)"><path data-c="1D458" d="M121 647Q121 657 125 670T137 683Q138 683 209 688T282 694Q294 694 294 686 294 679 244 477 194 279 194 272 213 282 223 291 247 309 292 354T362 415Q402 442 438 442 468 442 485 423T503 369Q503 344 496 327T477 302 456 291 438 288Q418 288 406 299T394 328Q394 353 410 369T442 390L458 393Q446 405 434 405H430Q398 402 367 380T294 316 228 255Q230 254 243 252T267 246 293 238 320 224 342 206 359 180 365 147Q365 130 360 106T354 66Q354 26 381 26 429 26 459 145 461 153 479 153H483Q499 153 499 144 499 139 496 130 455-11 378-11 333-11 305 15T277 90Q277 108 280 121T283 145Q283 167 269 183T234 206 200 217 182 220H180Q168 178 159 139T145 81 136 44 129 20 122 7 111-2Q98-11 83-11 66-11 57-1T48 16Q48 26 85 176T158 471L195 616Q196 629 188 632T149 637H144Q134 637 131 637T124 640 121 647Z"/></g></g><g data-mml-node="munder" transform="translate(4998.2,0)"><g data-mml-node="mo" transform="translate(84.4,0)"><path data-c="2211" d="M60 948Q63 950 665 950H1267L1325 815Q1384 677 1388 669H1348L1341 683Q1320 724 1285 761 1235 809 1174 838T1033 881 882 898 699 902H574 543 251L259 891Q722 258 724 252 725 250 724 246 721 243 460-56L196-356Q196-357 407-357 459-357 548-357T676-358Q812-358 896-353T1063-332 1204-283 1307-196Q1328-170 1348-124H1388Q1388-125 1381-145T1356-210 1325-294L1267-449 666-450Q64-450 61-448 55-446 55-439 55-437 57-433L590 177Q590 178 557 222T452 366 322 544L56 909 55 924Q55 945 60 948Z"/></g><g data-mml-node="TeXAtom" transform="translate(0,-1115.5) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D465" d="M52 289Q59 331 106 386T222 442Q257 442 286 424T329 379Q371 442 430 442 467 442 494 420T522 361Q522 332 508 314T481 292 458 288Q439 288 427 299T415 328Q415 374 465 391 454 404 425 404 412 404 406 402 368 386 350 336 290 115 290 78 290 50 306 38T341 26Q378 26 414 59T463 140Q466 150 469 151T485 153H489Q504 153 504 145 504 144 502 134 486 77 440 33T333-11Q263-11 227 52 186-10 133-10H127Q78-10 57 16T35 71Q35 103 54 123T99 143Q142 143 142 101 142 81 130 66T107 46 94 41L91 40Q91 39 97 36T113 29 132 26Q168 26 194 71 203 87 217 139T245 247 261 313Q266 340 266 352 266 380 251 392T217 404Q177 404 142 372T93 290Q91 281 88 280T72 278H58Q52 284 52 289Z"/></g><g data-mml-node="mo" transform="translate(572,0)"><path data-c="2208" d="M84 250Q84 372 166 450T360 539Q361 539 377 539T419 540 469 540H568Q583 532 583 520 583 511 570 501L466 500Q355 499 329 494 280 482 242 458T183 409 147 354 129 306 124 272V270H568Q583 262 583 250T568 230H124V228Q124 207 134 177T167 112 231 48 328 7Q355 1 466 0H570Q583-10 583-20 583-32 568-40H471Q464-40 446-40T417-41Q262-41 172 45 84 127 84 250Z"/></g><g data-mml-node="msub" transform="translate(1239,0)"><g data-mml-node="mi"><path data-c="1D436" d="M50 252Q50 367 117 473T286 641 490 704Q580 704 633 653 642 643 648 636T656 626L657 623Q660 623 684 649 691 655 699 663T715 679 725 690L740 705H746Q760 705 760 698 760 694 728 561 692 422 692 421 690 416 687 415T669 413H653Q647 419 647 422 647 423 648 429T650 449 651 481Q651 552 619 605T510 659Q484 659 454 652T382 628 299 572 226 479Q194 422 175 346T156 222Q156 108 232 58 280 24 350 24 441 24 512 92T606 240Q610 253 612 255T628 257Q648 257 648 248 648 243 647 239 618 132 523 55T319-22Q206-22 128 53T50 252Z"/></g><g data-mml-node="mi" transform="translate(748,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g></g></g><g data-mml-node="TeXAtom" data-mjx-texclass="ORD" transform="translate(6777.8,0)"><g data-mml-node="mo"><path data-c="2016" d="M133 736Q138 750 153 750 164 750 170 739 172 735 172 250T170-239Q164-250 152-250 144-250 138-244L137-243Q133-241 133-179T132 250Q132 731 133 736ZM329 739Q334 750 346 750 353 750 361 744L362 743Q366 741 366 679T367 250 367-178 362-243L361-244Q355-250 347-250 335-250 329-239 327-235 327 250T329 739Z"/></g></g><g data-mml-node="mi" transform="translate(7277.8,0)"><path data-c="1D465" d="M52 289Q59 331 106 386T222 442Q257 442 286 424T329 379Q371 442 430 442 467 442 494 420T522 361Q522 332 508 314T481 292 458 288Q439 288 427 299T415 328Q415 374 465 391 454 404 425 404 412 404 406 402 368 386 350 336 290 115 290 78 290 50 306 38T341 26Q378 26 414 59T463 140Q466 150 469 151T485 153H489Q504 153 504 145 504 144 502 134 486 77 440 33T333-11Q263-11 227 52 186-10 133-10H127Q78-10 57 16T35 71Q35 103 54 123T99 143Q142 143 142 101 142 81 130 66T107 46 94 41L91 40Q91 39 97 36T113 29 132 26Q168 26 194 71 203 87 217 139T245 247 261 313Q266 340 266 352 266 380 251 392T217 404Q177 404 142 372T93 290Q91 281 88 280T72 278H58Q52 284 52 289Z"/></g><g data-mml-node="mo" transform="translate(8072,0)"><path data-c="2212" d="M84 237T84 250 98 270H679Q694 262 694 250T679 230H98Q84 237 84 250Z"/></g><g data-mml-node="msub" transform="translate(9072.2,0)"><g data-mml-node="mi"><path data-c="1D462" d="M21 287Q21 295 30 318T55 370 99 420 158 442Q204 442 227 417T250 358Q250 340 216 246T182 105Q182 62 196 45T238 27 291 44 328 78L339 95Q341 99 377 247 407 367 413 387T427 416Q444 431 463 431 480 431 488 421T496 402L420 84Q419 79 419 68 419 43 426 35T447 26Q469 29 482 57T512 145Q514 153 532 153 551 153 551 144 550 139 549 130T540 98 523 55 498 17 462-8Q454-10 438-10 372-10 347 46 345 45 336 36T318 21 296 6 267-6 233-11Q189-11 155 7 103 38 103 113 103 170 138 262T173 379Q173 380 173 381 173 390 173 393T169 400 158 404H154Q131 404 112 385T82 344 65 302 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(605,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g><g data-mml-node="msubsup" transform="translate(9971.2,0)"><g data-mml-node="TeXAtom" data-mjx-texclass="ORD"><g data-mml-node="mo"><path data-c="2016" d="M133 736Q138 750 153 750 164 750 170 739 172 735 172 250T170-239Q164-250 152-250 144-250 138-244L137-243Q133-241 133-179T132 250Q132 731 133 736ZM329 739Q334 750 346 750 353 750 361 744L362 743Q366 741 366 679T367 250 367-178 362-243L361-244Q355-250 347-250 335-250 329-239 327-235 327 250T329 739Z"/></g></g><g data-mml-node="mn" transform="translate(533,413) scale(0.707)"><path data-c="32" d="M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315 301 241Q265 210 201 149L142 93 218 92Q375 92 385 97 392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19 31Q50 38 56 46T86 81Q115 113 136 137 145 147 170 174T204 211 233 244 261 278 284 308 305 340 320 369 333 401 340 431 343 464Q343 527 309 573T212 619Q179 619 154 602T119 569 109 550Q109 549 114 549 132 549 151 535T170 489Q170 464 154 447T109 429Z"/></g><g data-mml-node="mn" transform="translate(533,-247) scale(0.707)"><path data-c="32" d="M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315 301 241Q265 210 201 149L142 93 218 92Q375 92 385 97 392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19 31Q50 38 56 46T86 81Q115 113 136 137 145 147 170 174T204 211 233 244 261 278 284 308 305 340 320 369 333 401 340 431 343 464Q343 527 309 573T212 619Q179 619 154 602T119 569 109 550Q109 549 114 549 132 549 151 535T170 489Q170 464 154 447T109 429Z"/></g></g></g></g></g></svg><svg data-labels="true" preserveAspectRatio="xMaxYMid" viewBox="1278 -1783.9 1 3067.8"><g data-labels="true" transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mtd" id="mjx-eqn:1" transform="translate(0,793.2)"><text data-id-align="true"/><g data-idbox="true" transform="translate(0,-750)"><g data-mml-node="mtext"><path data-c="28" d="M94 250Q94 319 104 381T127 488 164 576 202 643 244 695 277 729 302 750H315 319Q333 750 333 741 333 738 316 720T275 667 226 581 184 443 167 250 184 58 225-81 274-167 316-220 333-241Q333-250 318-250H315 302L274-226Q180-141 137-14T94 250Z"/><path data-c="31" d="M213 578 200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641 273 663Q275 666 285 666 294 666 302 660V361L303 61Q310 54 315 52T339 48 401 46H427V0H416Q395 3 257 3 121 3 100 0H88V46H114Q136 46 152 46T177 47 193 50 201 52 207 57 213 61V578Z" transform="translate(389,0)"/><path data-c="29" d="M60 749 64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12 224-76 186-143 145-194 113-227 90-246Q87-249 86-250H74Q66-250 63-250T58-247 55-238Q56-237 66-225 221-64 221 250T66 725Q56 737 55 738 55 746 60 749Z" transform="translate(889,0)"/></g></g></g></g></svg></g></g></g></g></svg></mjx-container></p><p>其中<mjx-container class="MathJax" jax="SVG"><svg style="vertical-align:-.357ex" xmlns="http://www.w3.org/2000/svg" width="2.034ex" height="1.357ex" role="img" focusable="false" viewBox="0 -442 899 599.8"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(1,-1)"><g data-mml-node="math"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D462" d="M21 287Q21 295 30 318T55 370 99 420 158 442Q204 442 227 417T250 358Q250 340 216 246T182 105Q182 62 196 45T238 27 291 44 328 78L339 95Q341 99 377 247 407 367 413 387T427 416Q444 431 463 431 480 431 488 421T496 402L420 84Q419 79 419 68 419 43 426 35T447 26Q469 29 482 57T512 145Q514 153 532 153 551 153 551 144 550 139 549 130T540 98 523 55 498 17 462-8Q454-10 438-10 372-10 347 46 345 45 336 36T318 21 296 6 267-6 233-11Q189-11 155 7 103 38 103 113 103 170 138 262T173 379Q173 380 173 381 173 390 173 393T169 400 158 404H154Q131 404 112 385T82 344 65 302 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(605,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g></g></g></svg></mjx-container>是簇<mjx-container class="MathJax" jax="SVG"><svg style="vertical-align:-.357ex" xmlns="http://www.w3.org/2000/svg" width="2.357ex" height="1.952ex" role="img" focusable="false" viewBox="0 -705 1042 862.8"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(1,-1)"><g data-mml-node="math"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D436" d="M50 252Q50 367 117 473T286 641 490 704Q580 704 633 653 642 643 648 636T656 626L657 623Q660 623 684 649 691 655 699 663T715 679 725 690L740 705H746Q760 705 760 698 760 694 728 561 692 422 692 421 690 416 687 415T669 413H653Q647 419 647 422 647 423 648 429T650 449 651 481Q651 552 619 605T510 659Q484 659 454 652T382 628 299 572 226 479Q194 422 175 346T156 222Q156 108 232 58 280 24 350 24 441 24 512 92T606 240Q610 253 612 255T628 257Q648 257 648 248 648 243 647 239 618 132 523 55T319-22Q206-22 128 53T50 252Z"/></g><g data-mml-node="mi" transform="translate(748,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g></g></g></svg></mjx-container>的均值向量，也称为质心，表达式为：</p><p><mjx-container class="MathJax" jax="SVG" display="true" width="full" style="min-width:24.762ex"><svg style="vertical-align:-2.454ex;min-width:24.762ex" xmlns="http://www.w3.org/2000/svg" width="100%" height="6.039ex" role="img" focusable="false"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(0.0181,-0.0181) translate(0, -1584.5)"><g data-mml-node="math"><g data-mml-node="mtable" transform="translate(2078,0) translate(-2078,0)"><g transform="translate(0 1584.5) matrix(1 0 0 -1 0 0) scale(55.25)"><svg data-table="true" preserveAspectRatio="xMidYMid" viewBox="3394.3 -1584.5 1 2669.1"><g transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mlabeledtr" transform="translate(0,242.5)"><g data-mml-node="mtd"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D462" d="M21 287Q21 295 30 318T55 370 99 420 158 442Q204 442 227 417T250 358Q250 340 216 246T182 105Q182 62 196 45T238 27 291 44 328 78L339 95Q341 99 377 247 407 367 413 387T427 416Q444 431 463 431 480 431 488 421T496 402L420 84Q419 79 419 68 419 43 426 35T447 26Q469 29 482 57T512 145Q514 153 532 153 551 153 551 144 550 139 549 130T540 98 523 55 498 17 462-8Q454-10 438-10 372-10 347 46 345 45 336 36T318 21 296 6 267-6 233-11Q189-11 155 7 103 38 103 113 103 170 138 262T173 379Q173 380 173 381 173 390 173 393T169 400 158 404H154Q131 404 112 385T82 344 65 302 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(605,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g><g data-mml-node="mo" transform="translate(1176.7,0)"><path data-c="3D" d="M56 347Q56 360 70 367H707Q722 359 722 347 722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153 722 140 707 133H70Q56 140 56 153Z"/></g><g data-mml-node="mfrac" transform="translate(2232.5,0)"><g data-mml-node="mn" transform="translate(769,676)"><path data-c="31" d="M213 578 200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641 273 663Q275 666 285 666 294 666 302 660V361L303 61Q310 54 315 52T339 48 401 46H427V0H416Q395 3 257 3 121 3 100 0H88V46H114Q136 46 152 46T177 47 193 50 201 52 207 57 213 61V578Z"/></g><g data-mml-node="mrow" transform="translate(220,-709.5)"><g data-mml-node="mo" transform="translate(0 -0.5)"><path data-c="7C" d="M139-249H137Q125-249 119-235V251L120 737Q130 750 139 750 152 750 159 735V-235Q151-249 141-249H139Z"/></g><g data-mml-node="msub" transform="translate(278,0)"><g data-mml-node="mi"><path data-c="1D436" d="M50 252Q50 367 117 473T286 641 490 704Q580 704 633 653 642 643 648 636T656 626L657 623Q660 623 684 649 691 655 699 663T715 679 725 690L740 705H746Q760 705 760 698 760 694 728 561 692 422 692 421 690 416 687 415T669 413H653Q647 419 647 422 647 423 648 429T650 449 651 481Q651 552 619 605T510 659Q484 659 454 652T382 628 299 572 226 479Q194 422 175 346T156 222Q156 108 232 58 280 24 350 24 441 24 512 92T606 240Q610 253 612 255T628 257Q648 257 648 248 648 243 647 239 618 132 523 55T319-22Q206-22 128 53T50 252Z"/></g><g data-mml-node="mi" transform="translate(748,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g><g data-mml-node="mo" transform="translate(1320,0) translate(0 -0.5)"><path data-c="7C" d="M139-249H137Q125-249 119-235V251L120 737Q130 750 139 750 152 750 159 735V-235Q151-249 141-249H139Z"/></g></g><rect width="1798" height="60" x="120" y="220"/></g><g data-mml-node="munder" transform="translate(4437.1,0)"><g data-mml-node="mo" transform="translate(84.4,0)"><path data-c="2211" d="M60 948Q63 950 665 950H1267L1325 815Q1384 677 1388 669H1348L1341 683Q1320 724 1285 761 1235 809 1174 838T1033 881 882 898 699 902H574 543 251L259 891Q722 258 724 252 725 250 724 246 721 243 460-56L196-356Q196-357 407-357 459-357 548-357T676-358Q812-358 896-353T1063-332 1204-283 1307-196Q1328-170 1348-124H1388Q1388-125 1381-145T1356-210 1325-294L1267-449 666-450Q64-450 61-448 55-446 55-439 55-437 57-433L590 177Q590 178 557 222T452 366 322 544L56 909 55 924Q55 945 60 948Z"/></g><g data-mml-node="TeXAtom" transform="translate(0,-1115.5) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D465" d="M52 289Q59 331 106 386T222 442Q257 442 286 424T329 379Q371 442 430 442 467 442 494 420T522 361Q522 332 508 314T481 292 458 288Q439 288 427 299T415 328Q415 374 465 391 454 404 425 404 412 404 406 402 368 386 350 336 290 115 290 78 290 50 306 38T341 26Q378 26 414 59T463 140Q466 150 469 151T485 153H489Q504 153 504 145 504 144 502 134 486 77 440 33T333-11Q263-11 227 52 186-10 133-10H127Q78-10 57 16T35 71Q35 103 54 123T99 143Q142 143 142 101 142 81 130 66T107 46 94 41L91 40Q91 39 97 36T113 29 132 26Q168 26 194 71 203 87 217 139T245 247 261 313Q266 340 266 352 266 380 251 392T217 404Q177 404 142 372T93 290Q91 281 88 280T72 278H58Q52 284 52 289Z"/></g><g data-mml-node="mo" transform="translate(572,0)"><path data-c="2208" d="M84 250Q84 372 166 450T360 539Q361 539 377 539T419 540 469 540H568Q583 532 583 520 583 511 570 501L466 500Q355 499 329 494 280 482 242 458T183 409 147 354 129 306 124 272V270H568Q583 262 583 250T568 230H124V228Q124 207 134 177T167 112 231 48 328 7Q355 1 466 0H570Q583-10 583-20 583-32 568-40H471Q464-40 446-40T417-41Q262-41 172 45 84 127 84 250Z"/></g><g data-mml-node="msub" transform="translate(1239,0)"><g data-mml-node="mi"><path data-c="1D436" d="M50 252Q50 367 117 473T286 641 490 704Q580 704 633 653 642 643 648 636T656 626L657 623Q660 623 684 649 691 655 699 663T715 679 725 690L740 705H746Q760 705 760 698 760 694 728 561 692 422 692 421 690 416 687 415T669 413H653Q647 419 647 422 647 423 648 429T650 449 651 481Q651 552 619 605T510 659Q484 659 454 652T382 628 299 572 226 479Q194 422 175 346T156 222Q156 108 232 58 280 24 350 24 441 24 512 92T606 240Q610 253 612 255T628 257Q648 257 648 248 648 243 647 239 618 132 523 55T319-22Q206-22 128 53T50 252Z"/></g><g data-mml-node="mi" transform="translate(748,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g></g></g><g data-mml-node="mi" transform="translate(6216.7,0)"><path data-c="1D465" d="M52 289Q59 331 106 386T222 442Q257 442 286 424T329 379Q371 442 430 442 467 442 494 420T522 361Q522 332 508 314T481 292 458 288Q439 288 427 299T415 328Q415 374 465 391 454 404 425 404 412 404 406 402 368 386 350 336 290 115 290 78 290 50 306 38T341 26Q378 26 414 59T463 140Q466 150 469 151T485 153H489Q504 153 504 145 504 144 502 134 486 77 440 33T333-11Q263-11 227 52 186-10 133-10H127Q78-10 57 16T35 71Q35 103 54 123T99 143Q142 143 142 101 142 81 130 66T107 46 94 41L91 40Q91 39 97 36T113 29 132 26Q168 26 194 71 203 87 217 139T245 247 261 313Q266 340 266 352 266 380 251 392T217 404Q177 404 142 372T93 290Q91 281 88 280T72 278H58Q52 284 52 289Z"/></g></g></g></g></svg><svg data-labels="true" preserveAspectRatio="xMaxYMid" viewBox="1278 -1584.5 1 2669.1"><g data-labels="true" transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mtd" id="mjx-eqn:2" transform="translate(0,992.5)"><text data-id-align="true"/><g data-idbox="true" transform="translate(0,-750)"><g data-mml-node="mtext"><path data-c="28" d="M94 250Q94 319 104 381T127 488 164 576 202 643 244 695 277 729 302 750H315 319Q333 750 333 741 333 738 316 720T275 667 226 581 184 443 167 250 184 58 225-81 274-167 316-220 333-241Q333-250 318-250H315 302L274-226Q180-141 137-14T94 250Z"/><path data-c="32" d="M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315 301 241Q265 210 201 149L142 93 218 92Q375 92 385 97 392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19 31Q50 38 56 46T86 81Q115 113 136 137 145 147 170 174T204 211 233 244 261 278 284 308 305 340 320 369 333 401 340 431 343 464Q343 527 309 573T212 619Q179 619 154 602T119 569 109 550Q109 549 114 549 132 549 151 535T170 489Q170 464 154 447T109 429Z" transform="translate(389,0)"/><path data-c="29" d="M60 749 64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12 224-76 186-143 145-194 113-227 90-246Q87-249 86-250H74Q66-250 63-250T58-247 55-238Q56-237 66-225 221-64 221 250T66 725Q56 737 55 738 55 746 60 749Z" transform="translate(889,0)"/></g></g></g></g></svg></g></g></g></g></svg></mjx-container></p><p>这是一个 NP 难题，因此只能采用启发式迭代方法。</p><p>K-Means 采用的启发式方式很简单，用下面一组图就可以形象的描述:</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651998626506/202082163228.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651998626506/202082163228.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="启发式迭代"></p><p>图 a 表达了初始的数据集，假设 k=2。在图 b 中，随机选择了两个 k 类所对应的类别质心，即图中的红色质心和蓝色质心，然后分别求样本中所有点到这两个质心的距离，并标记每个样本的类别为和该样本距离最小的质心的类别，如图 c 所示，经过计算样本和红色质心和蓝色质心的距离，得到了所有样本点的第一轮迭代后的类别。此时对当前标记为红色和蓝色的点分别求其新的质心，如图 d 所示，新的红色质心和蓝色质心的位置已经发生了变动。图 e 和图 f 重复了在图 c 和图 d 的过程，即将所有点的类别标记为距离最近的质心的类别并求新的质心。最终得到的两个类别如图 f。</p><h3 id="K-means聚类最优k值的选取（手肘法）"><a href="#K-means聚类最优k值的选取（手肘法）" class="headerlink" title="K-means聚类最优k值的选取（手肘法）"></a><strong>K-means 聚类最优 k 值的选取（手肘法）</strong></h3><p>手肘法的核心指标是 SSE (sum of the squared errors，误差平方和), 公式见上文。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651998667001/20208216327.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651998667001/20208216327.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="K-means聚类最优k值的选取（手肘法）"></p><p>核心思想是：随着聚类数 k 的增大，样本划分会更加精细，每个簇的聚合程度会逐渐提高，那么误差平方和 SSE 自然会逐渐变小。并且，当 k 小于真实聚类数时，由于 k 的增大会大幅增加每个簇的聚合程度，故 SSE 的下降幅度会很大，而当 k 到达真实聚类数时，再增加 k 所得到的聚合程度回报会迅速变小，所以 SSE 的下降幅度会骤减，然后随着 k 值的继续增大而趋于平缓，也就是说 SSE 和 k 的关系图是一个手肘的形状，而这个肘部对应的 k 值就是数据的真实聚类数。</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651998696086/202082163129.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651998696086/202082163129.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="K-means聚类最优k值的选取（手肘法）"></p><h3 id="特征的规范化"><a href="#特征的规范化" class="headerlink" title="特征的规范化"></a><strong>特征的规范化</strong></h3><p>去除数据的单位限制，将其转化为无量纲的纯数值，便于不同单位或量级的指标能够进行计算和比较。</p><p>1、数据的中心化</p><p>所谓数据的中心化是指数据集中的各项数据减去数据集的均值。</p><p>2、数据的标准化</p><p>所谓数据的标准化是指中心化之后的数据在除以数据集的标准差，即数据集中的各项数据减去数据集的均值再除以数据集的标准差。</p><p>特征的规范化可以通过将每个特征转换为标准得分来完成。这就是说用对每个特征值求平均，用每个特征值减去平均值，然后除以特征值的标准差，如下标准分计算公式所示：<br><mjx-container class="MathJax" jax="SVG" display="true" width="full" style="min-width:38.841ex"><svg style="vertical-align:-1.951ex;min-width:38.841ex" xmlns="http://www.w3.org/2000/svg" width="100%" height="5.033ex" role="img" focusable="false"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(0.0181,-0.0181) translate(0, -1362.4)"><g data-mml-node="math"><g data-mml-node="mtable" transform="translate(2078,0) translate(-2078,0)"><g transform="translate(0 1362.4) matrix(1 0 0 -1 0 0) scale(55.25)"><svg data-table="true" preserveAspectRatio="xMidYMid" viewBox="6505.9 -1362.4 1 2224.8"><g transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mlabeledtr" transform="translate(0,-18.6)"><g data-mml-node="mtd"><g data-mml-node="mi"><path data-c="1D45B" d="M21 287Q22 293 24 303T36 341 56 388 89 425 135 442Q171 442 195 424T225 390 231 369Q231 367 232 367L243 378Q304 442 382 442 436 442 469 415T503 336 465 179 427 52Q427 26 444 26 450 26 453 27 482 32 505 65T540 145Q542 153 560 153 580 153 580 145 580 144 576 130 568 101 554 73T508 17 439-10Q392-10 371 17T350 73Q350 92 386 193T423 345Q423 404 379 404H374Q288 404 229 303L222 291 189 157Q156 26 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 112 180T152 343Q153 348 153 366 153 405 129 405 91 405 66 305 60 285 60 284 58 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(600,0)"><path data-c="1D45C" d="M201-11Q126-11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441 333 441 341 440 354 437 367 433T402 417 438 387 464 338 476 268Q476 161 390 75T201-11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375 346 405 306 405 243 405 195 347 158 303 140 230T121 120Z"/></g><g data-mml-node="mi" transform="translate(1085,0)"><path data-c="1D45F" d="M21 287Q22 290 23 295T28 317 38 348 53 381 73 411 99 433 132 442Q161 442 183 430T214 408 225 388Q227 382 228 382T236 389Q284 441 347 441H350Q398 441 422 400 430 381 430 363 430 333 417 315T391 292 366 288Q346 288 334 299T322 328Q322 376 378 392 356 405 342 405 286 405 239 331 229 315 224 298T190 165Q156 25 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 114 189T154 366Q154 405 128 405 107 405 92 377T68 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(1536,0)"><path data-c="1D45A" d="M21 287Q22 293 24 303T36 341 56 388 88 425 132 442 175 435 205 417 221 395 229 376L231 369Q231 367 232 367L243 378Q303 442 384 442 401 442 415 440T441 433 460 423 475 411 485 398 493 385 497 373 500 364 502 357L510 367Q573 442 659 442 713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145 857 144 853 130 845 101 831 73T785 17 716-10Q669-10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291 466 157Q433 26 428 16 415-11 385-11 372-11 364-4T353 8 350 18Q350 29 384 161L420 307Q423 322 423 345 423 404 379 404H374Q288 404 229 303L222 291 189 157Q156 26 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 112 181 151 335 151 342 154 357 154 369 154 405 129 405 107 405 92 377T69 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(2414,0)"><path data-c="1D44E" d="M33 157Q33 258 109 349T280 441Q331 441 370 392 386 422 416 422 429 422 439 414T449 394Q449 381 412 234T374 68Q374 43 381 35T402 26Q411 27 422 35 443 55 463 131 469 151 473 152 475 153 483 153H487Q506 153 506 144 506 138 501 117T481 63 449 13Q436 0 417-8 409-10 393-10 359-10 336 5T306 36L300 51Q299 52 296 50 294 48 292 46 233-10 172-10 117-10 75 30T33 157ZM351 328Q351 334 346 350T323 385 277 405Q242 405 210 374T160 293Q131 214 119 129 119 126 119 118T118 106Q118 61 136 44T179 26Q217 26 254 59T298 110Q300 114 325 217T351 328Z"/></g><g data-mml-node="mi" transform="translate(2943,0)"><path data-c="1D459" d="M117 59Q117 26 142 26 179 26 205 131 211 151 215 152 217 153 225 153H229Q238 153 241 153T246 151 248 144Q247 138 245 128T234 90 214 43 183 6 137-11Q101-11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623 167 626 166 628T162 632 157 634 149 635 141 636 132 637 122 637Q112 637 109 637T101 638 95 641 94 647Q94 649 96 661 101 680 107 682T179 688Q194 689 213 690T243 693 254 694Q266 694 266 686 266 675 193 386T118 83Q118 81 118 75T117 65V59Z"/></g><g data-mml-node="mi" transform="translate(3241,0)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(3586,0)"><path data-c="1D467" d="M347 338Q337 338 294 349T231 360Q211 360 197 356T174 346 162 335 155 324L153 320Q150 317 138 317 117 317 117 325 117 330 120 339 133 378 163 406T229 440Q241 442 246 442 271 442 291 425T329 392 367 375Q389 375 411 408T434 441Q435 442 449 442H462Q468 436 468 434 468 430 463 420T449 399 432 377 418 358L411 349Q368 298 275 214T160 106L148 94 163 93Q185 93 227 82T290 71Q328 71 360 90T402 140Q406 149 409 151T424 153Q443 153 443 143 443 138 442 134 425 72 376 31T278-11Q252-11 232 6T193 40 155 57Q111 57 76-3 70-11 59-11H54 41Q35-5 35-2 35 13 93 84 132 129 225 214T340 322Q352 338 347 338Z"/></g><g data-mml-node="mi" transform="translate(4051,0)"><path data-c="1D452" d="M39 168Q39 225 58 272T107 350 174 402 244 433 307 442H310Q355 442 388 420T421 355Q421 265 310 237 261 224 176 223 139 223 138 221 138 219 132 186T125 128Q125 81 146 54T209 26 302 45 394 111Q403 121 406 121 410 121 419 112T429 98 420 82 390 55 344 24 281-1 205-11Q126-11 83 42T39 168ZM373 353Q367 405 305 405 272 405 244 391T199 357 170 316 154 280 149 261Q149 260 169 260 282 260 327 284T373 353Z"/></g><g data-mml-node="msub" transform="translate(4517,0)"><g data-mml-node="mi"><path data-c="1D451" d="M366 683Q367 683 438 688T511 694Q523 694 523 686 523 679 450 384T375 83 374 68Q374 26 402 26 411 27 422 35 443 55 463 131 469 151 473 152 475 153 483 153H487 491Q506 153 506 145 506 140 503 129 490 79 473 48T445 8 417-8Q409-10 393-10 359-10 336 5T306 36L300 51Q299 52 296 50 294 48 292 46 233-10 172-10 117-10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400 369 394 369 396 370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405 242 405 210 374T160 293Q131 214 119 129 119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109 352 326Z"/></g><g data-mml-node="mi" transform="translate(553,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g><g data-mml-node="mo" transform="translate(5641.7,0)"><path data-c="3D" d="M56 347Q56 360 70 367H707Q722 359 722 347 722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153 722 140 707 133H70Q56 140 56 153Z"/></g><g data-mml-node="mfrac" transform="translate(6697.5,0)"><g data-mml-node="mrow" transform="translate(220,676)"><g data-mml-node="mi"><path data-c="1D453" d="M118-162Q120-162 124-164T135-167 147-168Q160-168 171-155T187-126Q197-99 221 27T267 267 289 382V385H242Q195 385 192 387 188 390 188 397L195 425Q197 430 203 430T250 431Q298 431 298 432 298 434 307 482T319 540Q356 705 465 705 502 703 526 683T550 630Q550 594 529 578T487 561Q443 561 443 603 443 622 454 636T478 657L487 662Q471 668 457 668 445 668 434 658T419 630Q412 601 403 552T387 469 380 433Q380 431 435 431 480 431 487 430T498 424Q499 420 496 407T491 391Q489 386 482 386T428 385H372L349 263Q301 15 282-47 255-132 212-173 175-205 139-205 107-205 81-186T55-132Q55-95 76-78T118-61Q162-61 162-103 162-122 151-136T127-157L118-162Z"/></g><g data-mml-node="mi" transform="translate(550,0)"><path data-c="1D452" d="M39 168Q39 225 58 272T107 350 174 402 244 433 307 442H310Q355 442 388 420T421 355Q421 265 310 237 261 224 176 223 139 223 138 221 138 219 132 186T125 128Q125 81 146 54T209 26 302 45 394 111Q403 121 406 121 410 121 419 112T429 98 420 82 390 55 344 24 281-1 205-11Q126-11 83 42T39 168ZM373 353Q367 405 305 405 272 405 244 391T199 357 170 316 154 280 149 261Q149 260 169 260 282 260 327 284T373 353Z"/></g><g data-mml-node="mi" transform="translate(1016,0)"><path data-c="1D44E" d="M33 157Q33 258 109 349T280 441Q331 441 370 392 386 422 416 422 429 422 439 414T449 394Q449 381 412 234T374 68Q374 43 381 35T402 26Q411 27 422 35 443 55 463 131 469 151 473 152 475 153 483 153H487Q506 153 506 144 506 138 501 117T481 63 449 13Q436 0 417-8 409-10 393-10 359-10 336 5T306 36L300 51Q299 52 296 50 294 48 292 46 233-10 172-10 117-10 75 30T33 157ZM351 328Q351 334 346 350T323 385 277 405Q242 405 210 374T160 293Q131 214 119 129 119 126 119 118T118 106Q118 61 136 44T179 26Q217 26 254 59T298 110Q300 114 325 217T351 328Z"/></g><g data-mml-node="mi" transform="translate(1545,0)"><path data-c="1D461" d="M26 385Q19 392 19 395 19 399 22 411T27 425Q29 430 36 430T87 431H140L159 511Q162 522 166 540T173 566 179 586 187 603 197 615 211 624 229 626Q247 625 254 615T261 596Q261 589 252 549T232 470L222 433Q222 431 272 431H323Q330 424 330 420 330 398 317 385H210L174 240Q135 80 135 68 135 26 162 26 197 26 230 60T283 144Q285 150 288 151T303 153H307Q322 153 322 145 322 142 319 133 314 117 301 95T267 48 216 6 155-11Q125-11 98 4T59 56Q57 64 57 83V101L92 241Q127 382 128 383 128 385 77 385H26Z"/></g><g data-mml-node="mi" transform="translate(1906,0)"><path data-c="1D462" d="M21 287Q21 295 30 318T55 370 99 420 158 442Q204 442 227 417T250 358Q250 340 216 246T182 105Q182 62 196 45T238 27 291 44 328 78L339 95Q341 99 377 247 407 367 413 387T427 416Q444 431 463 431 480 431 488 421T496 402L420 84Q419 79 419 68 419 43 426 35T447 26Q469 29 482 57T512 145Q514 153 532 153 551 153 551 144 550 139 549 130T540 98 523 55 498 17 462-8Q454-10 438-10 372-10 347 46 345 45 336 36T318 21 296 6 267-6 233-11Q189-11 155 7 103 38 103 113 103 170 138 262T173 379Q173 380 173 381 173 390 173 393T169 400 158 404H154Q131 404 112 385T82 344 65 302 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(2478,0)"><path data-c="1D45F" d="M21 287Q22 290 23 295T28 317 38 348 53 381 73 411 99 433 132 442Q161 442 183 430T214 408 225 388Q227 382 228 382T236 389Q284 441 347 441H350Q398 441 422 400 430 381 430 363 430 333 417 315T391 292 366 288Q346 288 334 299T322 328Q322 376 378 392 356 405 342 405 286 405 239 331 229 315 224 298T190 165Q156 25 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 114 189T154 366Q154 405 128 405 107 405 92 377T68 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="msub" transform="translate(2929,0)"><g data-mml-node="mi"><path data-c="1D452" d="M39 168Q39 225 58 272T107 350 174 402 244 433 307 442H310Q355 442 388 420T421 355Q421 265 310 237 261 224 176 223 139 223 138 221 138 219 132 186T125 128Q125 81 146 54T209 26 302 45 394 111Q403 121 406 121 410 121 419 112T429 98 420 82 390 55 344 24 281-1 205-11Q126-11 83 42T39 168ZM373 353Q367 405 305 405 272 405 244 391T199 357 170 316 154 280 149 261Q149 260 169 260 282 260 327 284T373 353Z"/></g><g data-mml-node="mi" transform="translate(499,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g><g data-mml-node="mo" transform="translate(3944.2,0)"><path data-c="2212" d="M84 237T84 250 98 270H679Q694 262 694 250T679 230H98Q84 237 84 250Z"/></g><g data-mml-node="msub" transform="translate(4944.4,0)"><g data-mml-node="mi"><path data-c="1D707" d="M58-216Q44-216 34-208T23-186Q23-176 96 116T173 414Q186 442 219 442 231 441 239 435T249 423 251 413Q251 401 220 279T187 142Q185 131 185 107V99Q185 26 252 26 261 26 270 27T287 31 302 38 315 45 327 55 338 65 348 77 356 88 365 100L372 110 408 253Q444 395 448 404 461 431 491 431 504 431 512 424T523 412 525 402L449 84Q448 79 448 68 448 43 455 35T476 26Q485 27 496 35 517 55 537 131 543 151 547 152 549 153 557 153H561Q580 153 580 144 580 138 575 117T555 63 523 13Q510 0 491-8 483-10 467-10 446-10 429-4T402 11 385 29 376 44 374 51L368 45Q362 39 350 30T324 12 288-4 246-11Q199-11 153 12L129-85Q108-167 104-180T92-202Q76-216 58-216Z"/></g><g data-mml-node="mi" transform="translate(636,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g></g><g data-mml-node="msub" transform="translate(2771.7,-686)"><g data-mml-node="mi"><path data-c="1D6FF" d="M195 609Q195 656 227 686T302 717Q319 716 351 709T407 697 433 690Q451 682 451 662 451 644 438 628T403 612Q382 612 348 641T288 671 249 657 235 628Q235 584 334 463 401 379 401 292 401 169 340 80T205-10H198Q127-10 83 36T36 153Q36 286 151 382 191 413 252 434 252 435 245 449T230 481 214 521 201 566 195 609ZM112 130Q112 83 136 55T204 27Q233 27 256 51T291 111 309 178 316 232Q316 267 309 298T295 344 269 400L259 396Q215 381 183 342T137 256 118 179 112 130Z"/></g><g data-mml-node="mi" transform="translate(477,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g><rect width="6074.3" height="60" x="120" y="220"/></g></g></g></g></svg><svg data-labels="true" preserveAspectRatio="xMaxYMid" viewBox="1278 -1362.4 1 2224.8"><g data-labels="true" transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mtd" id="mjx-eqn:3" transform="translate(0,731.4)"><text data-id-align="true"/><g data-idbox="true" transform="translate(0,-750)"><g data-mml-node="mtext"><path data-c="28" d="M94 250Q94 319 104 381T127 488 164 576 202 643 244 695 277 729 302 750H315 319Q333 750 333 741 333 738 316 720T275 667 226 581 184 443 167 250 184 58 225-81 274-167 316-220 333-241Q333-250 318-250H315 302L274-226Q180-141 137-14T94 250Z"/><path data-c="33" d="M127 463Q100 463 85 480T69 524Q69 579 117 622T233 665Q268 665 277 664 351 652 390 611T430 522Q430 470 396 421T302 350L299 348Q299 347 308 345T337 336 375 315Q457 262 457 175 457 96 395 37T238-22Q158-22 100 21T42 130Q42 158 60 175T105 193Q133 193 151 175T169 130Q169 119 166 110T159 94 148 82 136 74 126 70 118 67L114 66Q165 21 238 21 293 21 321 74 338 107 338 175V195Q338 290 274 322 259 328 213 329L171 330 168 332Q166 335 166 348 166 366 174 366 202 366 232 371 266 376 294 413T322 525V533Q322 590 287 612 265 626 240 626 208 626 181 615T143 592 132 580H135Q138 579 143 578T153 573 165 566 175 555 183 540 186 520Q186 498 172 481T127 463Z" transform="translate(389,0)"/><path data-c="29" d="M60 749 64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12 224-76 186-143 145-194 113-227 90-246Q87-249 86-250H74Q66-250 63-250T58-247 55-238Q56-237 66-225 221-64 221 250T66 725Q56 737 55 738 55 746 60 749Z" transform="translate(889,0)"/></g></g></g></g></svg></g></g></g></g></svg></mjx-container></p><h3 id="类别型变量"><a href="#类别型变量" class="headerlink" title="类别型变量"></a><strong>类别型变量</strong></h3><p>类别型特征可以用 one-hot 编码转换为几个二元特征，这几个二元特征可以看成数值型维度。</p><p>使用 N 位状态寄存器来对 N 个状态进行编码，每个状态都由他独立的寄存器位，并且在任意时候，其中只有一位有效。</p><p>解决了分类器不好处理属性数据的问题；在一定程度上也起到了扩充特征的作用。</p><h3 id="聚类结果评价指标"><a href="#聚类结果评价指标" class="headerlink" title="聚类结果评价指标"></a><strong>聚类结果评价指标</strong></h3><h4 id="Entropy（熵）"><a href="#Entropy（熵）" class="headerlink" title="Entropy（熵）"></a><strong>Entropy（熵）</strong></h4><p>好的聚类应该和人工标签保持一致，大部分情况下，标签相同的数据点应聚在一起，而标签不同的数据点不应该在一起，并且簇内的数据点标签相同。熵值会变得很小。</p><p>对于一个聚类 i，首先计算聚类 i 中的成员（member）属于类（class）j 的概率<br><mjx-container class="MathJax" jax="SVG" display="true" width="full" style="min-width:19.652ex"><svg style="vertical-align:-1.742ex;min-width:19.652ex" xmlns="http://www.w3.org/2000/svg" width="100%" height="4.615ex" role="img" focusable="false"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(0.0181,-0.0181) translate(0, -1270)"><g data-mml-node="math"><g data-mml-node="mtable" transform="translate(2078,0) translate(-2078,0)"><g transform="translate(0 1270) matrix(1 0 0 -1 0 0) scale(55.25)"><svg data-table="true" preserveAspectRatio="xMidYMid" viewBox="2265.1 -1270 1 2040"><g transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mlabeledtr" transform="translate(0,73.8)"><g data-mml-node="mtd"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D443" d="M287 628Q287 635 230 637 206 637 199 638T192 648Q192 649 194 659 200 679 203 681T397 683Q587 682 600 680 664 669 707 631T751 530Q751 453 685 389 616 321 507 303 500 302 402 301H307L277 182Q247 66 247 59 247 55 248 54T255 50 272 48 305 46H336Q342 37 342 35 342 19 335 5 330 0 319 0 316 0 282 1T182 2Q120 2 87 2T51 1Q33 1 33 11 33 13 36 25 40 41 44 43T67 46Q94 46 127 49 141 52 146 61 149 65 218 339T287 628ZM645 554Q645 567 643 575T634 597 609 619 560 635Q553 636 480 637 463 637 445 637T416 636 404 636Q391 635 386 627 384 621 367 550T332 412 314 344Q314 342 395 342H407 430Q542 342 590 392 617 419 631 471T645 554Z"/></g><g data-mml-node="TeXAtom" transform="translate(675,-150) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(345,0)"><path data-c="1D457" d="M297 596Q297 627 318 644T361 661Q378 661 389 651T403 623Q403 595 384 576T340 557Q322 557 310 567T297 596ZM288 376Q288 405 262 405 240 405 220 393T185 362 161 325 144 293L137 279Q135 278 121 278H107Q101 284 101 286T105 299Q126 348 164 391T252 441Q253 441 260 441T272 442Q296 441 316 432 341 418 354 401T367 348V332L318 133Q267-67 264-75 246-125 194-164T75-204Q25-204 7-183T-12-137Q-12-110 7-91T53-71Q70-71 82-81T95-112Q95-148 63-167 69-168 77-168 111-168 139-140T182-74L193-32Q204 11 219 72T251 197 278 308 289 365Q289 372 288 376Z"/></g></g></g><g data-mml-node="mo" transform="translate(1538.1,0)"><path data-c="3D" d="M56 347Q56 360 70 367H707Q722 359 722 347 722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153 722 140 707 133H70Q56 140 56 153Z"/></g><g data-mml-node="mfrac" transform="translate(2593.8,0)"><g data-mml-node="msub" transform="translate(220,754.2)"><g data-mml-node="mi"><path data-c="1D45A" d="M21 287Q22 293 24 303T36 341 56 388 88 425 132 442 175 435 205 417 221 395 229 376L231 369Q231 367 232 367L243 378Q303 442 384 442 401 442 415 440T441 433 460 423 475 411 485 398 493 385 497 373 500 364 502 357L510 367Q573 442 659 442 713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145 857 144 853 130 845 101 831 73T785 17 716-10Q669-10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291 466 157Q433 26 428 16 415-11 385-11 372-11 364-4T353 8 350 18Q350 29 384 161L420 307Q423 322 423 345 423 404 379 404H374Q288 404 229 303L222 291 189 157Q156 26 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 112 181 151 335 151 342 154 357 154 369 154 405 129 405 107 405 92 377T69 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="TeXAtom" transform="translate(911,-150) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(345,0)"><path data-c="1D457" d="M297 596Q297 627 318 644T361 661Q378 661 389 651T403 623Q403 595 384 576T340 557Q322 557 310 567T297 596ZM288 376Q288 405 262 405 240 405 220 393T185 362 161 325 144 293L137 279Q135 278 121 278H107Q101 284 101 286T105 299Q126 348 164 391T252 441Q253 441 260 441T272 442Q296 441 316 432 341 418 354 401T367 348V332L318 133Q267-67 264-75 246-125 194-164T75-204Q25-204 7-183T-12-137Q-12-110 7-91T53-71Q70-71 82-81T95-112Q95-148 63-167 69-168 77-168 111-168 139-140T182-74L193-32Q204 11 219 72T251 197 278 308 289 365Q289 372 288 376Z"/></g></g></g><g data-mml-node="msub" transform="translate(365.7,-686)"><g data-mml-node="mi"><path data-c="1D45A" d="M21 287Q22 293 24 303T36 341 56 388 88 425 132 442 175 435 205 417 221 395 229 376L231 369Q231 367 232 367L243 378Q303 442 384 442 401 442 415 440T441 433 460 423 475 411 485 398 493 385 497 373 500 364 502 357L510 367Q573 442 659 442 713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145 857 144 853 130 845 101 831 73T785 17 716-10Q669-10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291 466 157Q433 26 428 16 415-11 385-11 372-11 364-4T353 8 350 18Q350 29 384 161L420 307Q423 322 423 345 423 404 379 404H374Q288 404 229 303L222 291 189 157Q156 26 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 112 181 151 335 151 342 154 357 154 369 154 405 129 405 107 405 92 377T69 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(911,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g><rect width="1696.3" height="60" x="120" y="220"/></g></g></g></g></svg><svg data-labels="true" preserveAspectRatio="xMaxYMid" viewBox="1278 -1270 1 2040"><g data-labels="true" transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mtd" id="mjx-eqn:4" transform="translate(0,823.8)"><text data-id-align="true"/><g data-idbox="true" transform="translate(0,-750)"><g data-mml-node="mtext"><path data-c="28" d="M94 250Q94 319 104 381T127 488 164 576 202 643 244 695 277 729 302 750H315 319Q333 750 333 741 333 738 316 720T275 667 226 581 184 443 167 250 184 58 225-81 274-167 316-220 333-241Q333-250 318-250H315 302L274-226Q180-141 137-14T94 250Z"/><path data-c="34" d="M462 0Q444 3 333 3 217 3 199 0H190V46H221Q241 46 248 46T265 48 279 53 286 61Q287 63 287 115V165H28V211L179 442Q332 674 334 675 336 677 355 677H373L379 671V211H471V165H379V114Q379 73 379 66T385 54Q393 47 442 46H471V0H462ZM293 211V545L74 212 183 211H293Z" transform="translate(389,0)"/><path data-c="29" d="M60 749 64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12 224-76 186-143 145-194 113-227 90-246Q87-249 86-250H74Q66-250 63-250T58-247 55-238Q56-237 66-225 221-64 221 250T66 725Q56 737 55 738 55 746 60 749Z" transform="translate(889,0)"/></g></g></g></g></svg></g></g></g></g></svg></mjx-container><br>其中<mjx-container class="MathJax" jax="SVG"><svg style="vertical-align:-.357ex" xmlns="http://www.w3.org/2000/svg" width="2.726ex" height="1.357ex" role="img" focusable="false" viewBox="0 -442 1205 599.8"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(1,-1)"><g data-mml-node="math"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D45A" d="M21 287Q22 293 24 303T36 341 56 388 88 425 132 442 175 435 205 417 221 395 229 376L231 369Q231 367 232 367L243 378Q303 442 384 442 401 442 415 440T441 433 460 423 475 411 485 398 493 385 497 373 500 364 502 357L510 367Q573 442 659 442 713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145 857 144 853 130 845 101 831 73T785 17 716-10Q669-10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291 466 157Q433 26 428 16 415-11 385-11 372-11 364-4T353 8 350 18Q350 29 384 161L420 307Q423 322 423 345 423 404 379 404H374Q288 404 229 303L222 291 189 157Q156 26 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 112 181 151 335 151 342 154 357 154 369 154 405 129 405 107 405 92 377T69 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="TeXAtom" transform="translate(911,-150) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g></g></g></g></svg></mjx-container>是在聚类 i 中所有成员的个数，<mjx-container class="MathJax" jax="SVG"><svg style="vertical-align:-.666ex" xmlns="http://www.w3.org/2000/svg" width="3.385ex" height="1.666ex" role="img" focusable="false" viewBox="0 -442 1496.3 736.2"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(1,-1)"><g data-mml-node="math"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D45A" d="M21 287Q22 293 24 303T36 341 56 388 88 425 132 442 175 435 205 417 221 395 229 376L231 369Q231 367 232 367L243 378Q303 442 384 442 401 442 415 440T441 433 460 423 475 411 485 398 493 385 497 373 500 364 502 357L510 367Q573 442 659 442 713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145 857 144 853 130 845 101 831 73T785 17 716-10Q669-10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291 466 157Q433 26 428 16 415-11 385-11 372-11 364-4T353 8 350 18Q350 29 384 161L420 307Q423 322 423 345 423 404 379 404H374Q288 404 229 303L222 291 189 157Q156 26 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 112 181 151 335 151 342 154 357 154 369 154 405 129 405 107 405 92 377T69 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="TeXAtom" transform="translate(911,-150) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(345,0)"><path data-c="1D457" d="M297 596Q297 627 318 644T361 661Q378 661 389 651T403 623Q403 595 384 576T340 557Q322 557 310 567T297 596ZM288 376Q288 405 262 405 240 405 220 393T185 362 161 325 144 293L137 279Q135 278 121 278H107Q101 284 101 286T105 299Q126 348 164 391T252 441Q253 441 260 441T272 442Q296 441 316 432 341 418 354 401T367 348V332L318 133Q267-67 264-75 246-125 194-164T75-204Q25-204 7-183T-12-137Q-12-110 7-91T53-71Q70-71 82-81T95-112Q95-148 63-167 69-168 77-168 111-168 139-140T182-74L193-32Q204 11 219 72T251 197 278 308 289 365Q289 372 288 376Z"/></g></g></g></g></g></svg></mjx-container>是聚类 i 中的成员属于类 j 的个数。</p><p>每个聚类的 entropy 可以表示为<br><mjx-container class="MathJax" jax="SVG" display="true" width="full" style="min-width:29.536ex"><svg style="vertical-align:-2.902ex;min-width:29.536ex" xmlns="http://www.w3.org/2000/svg" width="100%" height="6.935ex" role="img" focusable="false"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(0.0181,-0.0181) translate(0, -1782.6)"><g data-mml-node="math"><g data-mml-node="mtable" transform="translate(2078,0) translate(-2078,0)"><g transform="translate(0 1782.6) matrix(1 0 0 -1 0 0) scale(55.25)"><svg data-table="true" preserveAspectRatio="xMidYMid" viewBox="4449.5 -1782.6 1 3065.1"><g transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mlabeledtr" transform="translate(0,49.6)"><g data-mml-node="mtd"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D452" d="M39 168Q39 225 58 272T107 350 174 402 244 433 307 442H310Q355 442 388 420T421 355Q421 265 310 237 261 224 176 223 139 223 138 221 138 219 132 186T125 128Q125 81 146 54T209 26 302 45 394 111Q403 121 406 121 410 121 419 112T429 98 420 82 390 55 344 24 281-1 205-11Q126-11 83 42T39 168ZM373 353Q367 405 305 405 272 405 244 391T199 357 170 316 154 280 149 261Q149 260 169 260 282 260 327 284T373 353Z"/></g><g data-mml-node="mi" transform="translate(499,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g><g data-mml-node="mo" transform="translate(1070.7,0)"><path data-c="3D" d="M56 347Q56 360 70 367H707Q722 359 722 347 722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153 722 140 707 133H70Q56 140 56 153Z"/></g><g data-mml-node="mo" transform="translate(2126.5,0)"><path data-c="2212" d="M84 237T84 250 98 270H679Q694 262 694 250T679 230H98Q84 237 84 250Z"/></g><g data-mml-node="munderover" transform="translate(3071.2,0)"><g data-mml-node="mo"><path data-c="2211" d="M60 948Q63 950 665 950H1267L1325 815Q1384 677 1388 669H1348L1341 683Q1320 724 1285 761 1235 809 1174 838T1033 881 882 898 699 902H574 543 251L259 891Q722 258 724 252 725 250 724 246 721 243 460-56L196-356Q196-357 407-357 459-357 548-357T676-358Q812-358 896-353T1063-332 1204-283 1307-196Q1328-170 1348-124H1388Q1388-125 1381-145T1356-210 1325-294L1267-449 666-450Q64-450 61-448 55-446 55-439 55-437 57-433L590 177Q590 178 557 222T452 366 322 544L56 909 55 924Q55 945 60 948Z"/></g><g data-mml-node="TeXAtom" transform="translate(124.5,-1087.9) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D457" d="M297 596Q297 627 318 644T361 661Q378 661 389 651T403 623Q403 595 384 576T340 557Q322 557 310 567T297 596ZM288 376Q288 405 262 405 240 405 220 393T185 362 161 325 144 293L137 279Q135 278 121 278H107Q101 284 101 286T105 299Q126 348 164 391T252 441Q253 441 260 441T272 442Q296 441 316 432 341 418 354 401T367 348V332L318 133Q267-67 264-75 246-125 194-164T75-204Q25-204 7-183T-12-137Q-12-110 7-91T53-71Q70-71 82-81T95-112Q95-148 63-167 69-168 77-168 111-168 139-140T182-74L193-32Q204 11 219 72T251 197 278 308 289 365Q289 372 288 376Z"/></g><g data-mml-node="mo" transform="translate(412,0)"><path data-c="3D" d="M56 347Q56 360 70 367H707Q722 359 722 347 722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153 722 140 707 133H70Q56 140 56 153Z"/></g><g data-mml-node="mn" transform="translate(1190,0)"><path data-c="31" d="M213 578 200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641 273 663Q275 666 285 666 294 666 302 660V361L303 61Q310 54 315 52T339 48 401 46H427V0H416Q395 3 257 3 121 3 100 0H88V46H114Q136 46 152 46T177 47 193 50 201 52 207 57 213 61V578Z"/></g></g><g data-mml-node="TeXAtom" transform="translate(481.2,1150) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D43F" d="M228 637Q194 637 192 641 191 643 191 649 191 673 202 682 204 683 217 683 271 680 344 680 485 680 506 683H518Q524 677 524 674T522 656Q517 641 513 637H475Q406 636 394 628 387 624 380 600T313 336Q297 271 279 198T252 88L243 52Q243 48 252 48T311 46H328Q360 46 379 47T428 54 478 72 522 106 564 161Q580 191 594 228T611 270Q616 273 628 273H641Q647 264 647 262T627 203 583 83 557 9Q555 4 553 3T537 0 494-1Q483-1 418-1T294 0H116Q32 0 32 10 32 17 34 24 39 43 44 45 48 46 59 46H65Q92 46 125 49 139 52 144 61 147 65 216 339T285 628Q285 635 228 637Z"/></g></g></g><g data-mml-node="msub" transform="translate(4681.8,0)"><g data-mml-node="mi"><path data-c="1D443" d="M287 628Q287 635 230 637 206 637 199 638T192 648Q192 649 194 659 200 679 203 681T397 683Q587 682 600 680 664 669 707 631T751 530Q751 453 685 389 616 321 507 303 500 302 402 301H307L277 182Q247 66 247 59 247 55 248 54T255 50 272 48 305 46H336Q342 37 342 35 342 19 335 5 330 0 319 0 316 0 282 1T182 2Q120 2 87 2T51 1Q33 1 33 11 33 13 36 25 40 41 44 43T67 46Q94 46 127 49 141 52 146 61 149 65 218 339T287 628ZM645 554Q645 567 643 575T634 597 609 619 560 635Q553 636 480 637 463 637 445 637T416 636 404 636Q391 635 386 627 384 621 367 550T332 412 314 344Q314 342 395 342H407 430Q542 342 590 392 617 419 631 471T645 554Z"/></g><g data-mml-node="TeXAtom" transform="translate(675,-150) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(345,0)"><path data-c="1D457" d="M297 596Q297 627 318 644T361 661Q378 661 389 651T403 623Q403 595 384 576T340 557Q322 557 310 567T297 596ZM288 376Q288 405 262 405 240 405 220 393T185 362 161 325 144 293L137 279Q135 278 121 278H107Q101 284 101 286T105 299Q126 348 164 391T252 441Q253 441 260 441T272 442Q296 441 316 432 341 418 354 401T367 348V332L318 133Q267-67 264-75 246-125 194-164T75-204Q25-204 7-183T-12-137Q-12-110 7-91T53-71Q70-71 82-81T95-112Q95-148 63-167 69-168 77-168 111-168 139-140T182-74L193-32Q204 11 219 72T251 197 278 308 289 365Q289 372 288 376Z"/></g></g></g><g data-mml-node="mi" transform="translate(5942.1,0)"><path data-c="1D459" d="M117 59Q117 26 142 26 179 26 205 131 211 151 215 152 217 153 225 153H229Q238 153 241 153T246 151 248 144Q247 138 245 128T234 90 214 43 183 6 137-11Q101-11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623 167 626 166 628T162 632 157 634 149 635 141 636 132 637 122 637Q112 637 109 637T101 638 95 641 94 647Q94 649 96 661 101 680 107 682T179 688Q194 689 213 690T243 693 254 694Q266 694 266 686 266 675 193 386T118 83Q118 81 118 75T117 65V59Z"/></g><g data-mml-node="mi" transform="translate(6240.1,0)"><path data-c="1D45C" d="M201-11Q126-11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441 333 441 341 440 354 437 367 433T402 417 438 387 464 338 476 268Q476 161 390 75T201-11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375 346 405 306 405 243 405 195 347 158 303 140 230T121 120Z"/></g><g data-mml-node="msub" transform="translate(6725.1,0)"><g data-mml-node="mi"><path data-c="1D454" d="M311 43Q296 30 267 15T206 0Q143 0 105 45T66 160Q66 265 143 353T314 442Q361 442 401 394L404 398Q406 401 409 404T418 412 431 419 447 422Q461 422 470 413T480 394Q480 379 423 152T363-80Q345-134 286-169T151-205Q10-205 10-137 10-111 28-91T74-71Q89-71 102-80T116-111Q116-121 114-130T107-144 99-154 92-162L90-164H91Q101-167 151-167 189-167 211-155 234-144 254-122T282-75Q288-56 298-13 311 35 311 43ZM384 328 380 339Q377 350 375 354T369 368 359 382 346 393 328 402 306 405Q262 405 221 352 191 313 171 233T151 117Q151 38 213 38 269 38 323 108L331 118 384 328Z"/></g><g data-mml-node="mn" transform="translate(510,-150) scale(0.707)"><path data-c="32" d="M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315 301 241Q265 210 201 149L142 93 218 92Q375 92 385 97 392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19 31Q50 38 56 46T86 81Q115 113 136 137 145 147 170 174T204 211 233 244 261 278 284 308 305 340 320 369 333 401 340 431 343 464Q343 527 309 573T212 619Q179 619 154 602T119 569 109 550Q109 549 114 549 132 549 151 535T170 489Q170 464 154 447T109 429Z"/></g></g><g data-mml-node="msub" transform="translate(7638.7,0)"><g data-mml-node="mi"><path data-c="1D443" d="M287 628Q287 635 230 637 206 637 199 638T192 648Q192 649 194 659 200 679 203 681T397 683Q587 682 600 680 664 669 707 631T751 530Q751 453 685 389 616 321 507 303 500 302 402 301H307L277 182Q247 66 247 59 247 55 248 54T255 50 272 48 305 46H336Q342 37 342 35 342 19 335 5 330 0 319 0 316 0 282 1T182 2Q120 2 87 2T51 1Q33 1 33 11 33 13 36 25 40 41 44 43T67 46Q94 46 127 49 141 52 146 61 149 65 218 339T287 628ZM645 554Q645 567 643 575T634 597 609 619 560 635Q553 636 480 637 463 637 445 637T416 636 404 636Q391 635 386 627 384 621 367 550T332 412 314 344Q314 342 395 342H407 430Q542 342 590 392 617 419 631 471T645 554Z"/></g><g data-mml-node="TeXAtom" transform="translate(675,-150) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(345,0)"><path data-c="1D457" d="M297 596Q297 627 318 644T361 661Q378 661 389 651T403 623Q403 595 384 576T340 557Q322 557 310 567T297 596ZM288 376Q288 405 262 405 240 405 220 393T185 362 161 325 144 293L137 279Q135 278 121 278H107Q101 284 101 286T105 299Q126 348 164 391T252 441Q253 441 260 441T272 442Q296 441 316 432 341 418 354 401T367 348V332L318 133Q267-67 264-75 246-125 194-164T75-204Q25-204 7-183T-12-137Q-12-110 7-91T53-71Q70-71 82-81T95-112Q95-148 63-167 69-168 77-168 111-168 139-140T182-74L193-32Q204 11 219 72T251 197 278 308 289 365Q289 372 288 376Z"/></g></g></g></g></g></g></svg><svg data-labels="true" preserveAspectRatio="xMaxYMid" viewBox="1278 -1782.6 1 3065.1"><g data-labels="true" transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mtd" id="mjx-eqn:5" transform="translate(0,799.6)"><text data-id-align="true"/><g data-idbox="true" transform="translate(0,-750)"><g data-mml-node="mtext"><path data-c="28" d="M94 250Q94 319 104 381T127 488 164 576 202 643 244 695 277 729 302 750H315 319Q333 750 333 741 333 738 316 720T275 667 226 581 184 443 167 250 184 58 225-81 274-167 316-220 333-241Q333-250 318-250H315 302L274-226Q180-141 137-14T94 250Z"/><path data-c="35" d="M164 157Q164 133 148 117T109 101H102Q148 22 224 22 294 22 326 82 345 115 345 210 345 313 318 349 292 382 260 382H254Q176 382 136 314 132 307 129 306T114 304Q97 304 95 310 93 314 93 485V614Q93 664 98 664 100 666 102 666 103 666 123 658T178 642 253 634Q324 634 389 662 397 666 402 666 410 666 410 648V635Q328 538 205 538 174 538 149 544L139 546V374Q158 388 169 396T205 412 256 420Q337 420 393 355T449 201Q449 109 385 44T229-22Q148-22 99 32T50 154Q50 178 61 192T84 210 107 214Q132 214 148 197T164 157Z" transform="translate(389,0)"/><path data-c="29" d="M60 749 64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12 224-76 186-143 145-194 113-227 90-246Q87-249 86-250H74Q66-250 63-250T58-247 55-238Q56-237 66-225 221-64 221 250T66 725Q56 737 55 738 55 746 60 749Z" transform="translate(889,0)"/></g></g></g></g></svg></g></g></g></g></svg></mjx-container><br>其中 L 是类（class）的个数。</p><p>整个聚类划分的 entropy 为<br><mjx-container class="MathJax" jax="SVG" display="true" width="full" style="min-width:22.634ex"><svg style="vertical-align:-2.804ex;min-width:22.634ex" xmlns="http://www.w3.org/2000/svg" width="100%" height="6.74ex" role="img" focusable="false"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(0.0181,-0.0181) translate(0, -1739.4)"><g data-mml-node="math"><g data-mml-node="mtable" transform="translate(2078,0) translate(-2078,0)"><g transform="translate(0 1739.4) matrix(1 0 0 -1 0 0) scale(55.25)"><svg data-table="true" preserveAspectRatio="xMidYMid" viewBox="2924.1 -1739.4 1 2978.9"><g transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mlabeledtr" transform="translate(0,6.5)"><g data-mml-node="mtd"><g data-mml-node="mi"><path data-c="1D452" d="M39 168Q39 225 58 272T107 350 174 402 244 433 307 442H310Q355 442 388 420T421 355Q421 265 310 237 261 224 176 223 139 223 138 221 138 219 132 186T125 128Q125 81 146 54T209 26 302 45 394 111Q403 121 406 121 410 121 419 112T429 98 420 82 390 55 344 24 281-1 205-11Q126-11 83 42T39 168ZM373 353Q367 405 305 405 272 405 244 391T199 357 170 316 154 280 149 261Q149 260 169 260 282 260 327 284T373 353Z"/></g><g data-mml-node="mo" transform="translate(743.8,0)"><path data-c="3D" d="M56 347Q56 360 70 367H707Q722 359 722 347 722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153 722 140 707 133H70Q56 140 56 153Z"/></g><g data-mml-node="munderover" transform="translate(1799.6,0)"><g data-mml-node="mo"><path data-c="2211" d="M60 948Q63 950 665 950H1267L1325 815Q1384 677 1388 669H1348L1341 683Q1320 724 1285 761 1235 809 1174 838T1033 881 882 898 699 902H574 543 251L259 891Q722 258 724 252 725 250 724 246 721 243 460-56L196-356Q196-357 407-357 459-357 548-357T676-358Q812-358 896-353T1063-332 1204-283 1307-196Q1328-170 1348-124H1388Q1388-125 1381-145T1356-210 1325-294L1267-449 666-450Q64-450 61-448 55-446 55-439 55-437 57-433L590 177Q590 178 557 222T452 366 322 544L56 909 55 924Q55 945 60 948Z"/></g><g data-mml-node="TeXAtom" transform="translate(148.2,-1087.9) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mo" transform="translate(345,0)"><path data-c="3D" d="M56 347Q56 360 70 367H707Q722 359 722 347 722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153 722 140 707 133H70Q56 140 56 153Z"/></g><g data-mml-node="mn" transform="translate(1123,0)"><path data-c="31" d="M213 578 200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641 273 663Q275 666 285 666 294 666 302 660V361L303 61Q310 54 315 52T339 48 401 46H427V0H416Q395 3 257 3 121 3 100 0H88V46H114Q136 46 152 46T177 47 193 50 201 52 207 57 213 61V578Z"/></g></g><g data-mml-node="TeXAtom" transform="translate(407.7,1150) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D43E" d="M285 628Q285 635 228 637 205 637 198 638T191 647Q191 649 193 661 199 681 203 682 205 683 214 683H219Q260 681 355 681 389 681 418 681T463 682 483 682Q500 682 500 674 500 669 497 660 496 658 496 654T495 648 493 644 490 641 486 639 479 638 470 637 456 637Q416 636 405 634T387 623L306 305Q307 305 490 449T678 597Q692 611 692 620 692 635 667 637 651 637 651 648 651 650 654 662T659 677Q662 682 676 682 680 682 711 681T791 680Q814 680 839 681T869 682Q889 682 889 672 889 650 881 642 878 637 862 637 787 632 726 586 710 576 656 534T556 455L509 418 518 396Q527 374 546 329T581 244Q656 67 661 61 663 59 666 57 680 47 717 46H738Q744 38 744 37T741 19Q737 6 731 0H720Q680 3 625 3 503 3 488 0H478Q472 6 472 9T474 27Q478 40 480 43T491 46H494Q544 46 544 71 544 75 517 141T485 216L427 354 359 301 291 248 268 155Q245 63 245 58 245 51 253 49T303 46H334Q340 37 340 35 340 19 333 5 328 0 317 0 314 0 280 1T180 2Q118 2 85 2T49 1Q31 1 31 11 31 13 34 25 38 41 42 43T65 46Q92 46 125 49 139 52 144 61 147 65 216 339T285 628Z"/></g></g></g><g data-mml-node="mfrac" transform="translate(3410.2,0)"><g data-mml-node="msub" transform="translate(220,676)"><g data-mml-node="mi"><path data-c="1D45A" d="M21 287Q22 293 24 303T36 341 56 388 88 425 132 442 175 435 205 417 221 395 229 376L231 369Q231 367 232 367L243 378Q303 442 384 442 401 442 415 440T441 433 460 423 475 411 485 398 493 385 497 373 500 364 502 357L510 367Q573 442 659 442 713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145 857 144 853 130 845 101 831 73T785 17 716-10Q669-10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291 466 157Q433 26 428 16 415-11 385-11 372-11 364-4T353 8 350 18Q350 29 384 161L420 307Q423 322 423 345 423 404 379 404H374Q288 404 229 303L222 291 189 157Q156 26 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 112 181 151 335 151 342 154 357 154 369 154 405 129 405 107 405 92 377T69 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><g data-mml-node="mi" transform="translate(911,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g><g data-mml-node="mi" transform="translate(383.5,-686)"><path data-c="1D45A" d="M21 287Q22 293 24 303T36 341 56 388 88 425 132 442 175 435 205 417 221 395 229 376L231 369Q231 367 232 367L243 378Q303 442 384 442 401 442 415 440T441 433 460 423 475 411 485 398 493 385 497 373 500 364 502 357L510 367Q573 442 659 442 713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145 857 144 853 130 845 101 831 73T785 17 716-10Q669-10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291 466 157Q433 26 428 16 415-11 385-11 372-11 364-4T353 8 350 18Q350 29 384 161L420 307Q423 322 423 345 423 404 379 404H374Q288 404 229 303L222 291 189 157Q156 26 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 112 181 151 335 151 342 154 357 154 369 154 405 129 405 107 405 92 377T69 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g><rect width="1405" height="60" x="120" y="220"/></g><g data-mml-node="msub" transform="translate(5055.2,0)"><g data-mml-node="mi"><path data-c="1D452" d="M39 168Q39 225 58 272T107 350 174 402 244 433 307 442H310Q355 442 388 420T421 355Q421 265 310 237 261 224 176 223 139 223 138 221 138 219 132 186T125 128Q125 81 146 54T209 26 302 45 394 111Q403 121 406 121 410 121 419 112T429 98 420 82 390 55 344 24 281-1 205-11Q126-11 83 42T39 168ZM373 353Q367 405 305 405 272 405 244 391T199 357 170 316 154 280 149 261Q149 260 169 260 282 260 327 284T373 353Z"/></g><g data-mml-node="mi" transform="translate(499,-150) scale(0.707)"><path data-c="1D456" d="M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369 98 420 158 442Q197 442 223 419T250 357Q250 340 236 301T196 196 154 83Q149 61 149 51 149 26 166 26 175 26 185 29T208 43 235 78 260 137Q263 149 265 151T282 153Q302 153 302 143 302 135 293 112T268 61 223 11 161-11Q129-11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281 56 279 53 278 49 278 41 278H27Q21 284 21 287Z"/></g></g></g></g></g></svg><svg data-labels="true" preserveAspectRatio="xMaxYMid" viewBox="1278 -1739.4 1 2978.9"><g data-labels="true" transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mtd" id="mjx-eqn:6" transform="translate(0,756.5)"><text data-id-align="true"/><g data-idbox="true" transform="translate(0,-750)"><g data-mml-node="mtext"><path data-c="28" d="M94 250Q94 319 104 381T127 488 164 576 202 643 244 695 277 729 302 750H315 319Q333 750 333 741 333 738 316 720T275 667 226 581 184 443 167 250 184 58 225-81 274-167 316-220 333-241Q333-250 318-250H315 302L274-226Q180-141 137-14T94 250Z"/><path data-c="36" d="M42 313Q42 476 123 571T303 666Q372 666 402 630T432 550Q432 525 418 510T379 495Q356 495 341 509T326 548Q326 592 373 601 351 623 311 626 240 626 194 566 147 500 147 364L148 360Q153 366 156 373 197 433 263 433H267Q313 433 348 414 372 400 396 374T435 317Q456 268 456 210V192Q456 169 451 149 440 90 387 34T253-22Q225-22 199-14T143 16 92 75 56 172 42 313ZM257 397Q227 397 205 380T171 335 154 278 148 216Q148 133 160 97T198 39Q222 21 251 21 302 21 329 59 342 77 347 104T352 209Q352 289 347 316T329 361Q302 397 257 397Z" transform="translate(389,0)"/><path data-c="29" d="M60 749 64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12 224-76 186-143 145-194 113-227 90-246Q87-249 86-250H74Q66-250 63-250T58-247 55-238Q56-237 66-225 221-64 221 250T66 725Q56 737 55 738 55 746 60 749Z" transform="translate(889,0)"/></g></g></g></g></svg></g></g></g></g></svg></mjx-container><br>其中 K 是聚类（cluster）的数目，m 是整个聚类划分所涉及到的成员个数。</p><h4 id="Accuracy-准确率"><a href="#Accuracy-准确率" class="headerlink" title="Accuracy(准确率)"></a><strong>Accuracy (准确率)</strong></h4><p>比较每一条聚类结果是否和真的结果一致.</p><p><mjx-container class="MathJax" jax="SVG" display="true" width="full" style="min-width:20.766ex"><svg style="vertical-align:-1.748ex;min-width:20.766ex" xmlns="http://www.w3.org/2000/svg" width="100%" height="4.627ex" role="img" focusable="false"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(0.0181,-0.0181) translate(0, -1272.5)"><g data-mml-node="math"><g data-mml-node="mtable" transform="translate(2078,0) translate(-2078,0)"><g transform="translate(0 1272.5) matrix(1 0 0 -1 0 0) scale(55.25)"><svg data-table="true" preserveAspectRatio="xMidYMid" viewBox="2511.3 -1272.5 1 2045"><g transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mlabeledtr" transform="translate(0,-86.5)"><g data-mml-node="mtd"><g data-mml-node="mi"><path data-c="1D44E" d="M33 157Q33 258 109 349T280 441Q331 441 370 392 386 422 416 422 429 422 439 414T449 394Q449 381 412 234T374 68Q374 43 381 35T402 26Q411 27 422 35 443 55 463 131 469 151 473 152 475 153 483 153H487Q506 153 506 144 506 138 501 117T481 63 449 13Q436 0 417-8 409-10 393-10 359-10 336 5T306 36L300 51Q299 52 296 50 294 48 292 46 233-10 172-10 117-10 75 30T33 157ZM351 328Q351 334 346 350T323 385 277 405Q242 405 210 374T160 293Q131 214 119 129 119 126 119 118T118 106Q118 61 136 44T179 26Q217 26 254 59T298 110Q300 114 325 217T351 328Z"/></g><g data-mml-node="mi" transform="translate(529,0)"><path data-c="1D450" d="M34 159Q34 268 120 355T306 442Q362 442 394 418T427 355Q427 326 408 306T360 285Q341 285 330 295T319 325 330 359 352 380 366 386H367Q367 388 361 392T340 400 306 404Q276 404 249 390 228 381 206 359 162 315 142 235T121 119Q121 73 147 50 169 26 205 26H209Q321 26 394 111 403 121 406 121 410 121 419 112T429 98 420 83 391 55 346 25 282 0 202-11Q127-11 81 37T34 159Z"/></g><g data-mml-node="mi" transform="translate(962,0)"><path data-c="1D450" d="M34 159Q34 268 120 355T306 442Q362 442 394 418T427 355Q427 326 408 306T360 285Q341 285 330 295T319 325 330 359 352 380 366 386H367Q367 388 361 392T340 400 306 404Q276 404 249 390 228 381 206 359 162 315 142 235T121 119Q121 73 147 50 169 26 205 26H209Q321 26 394 111 403 121 406 121 410 121 419 112T429 98 420 83 391 55 346 25 282 0 202-11Q127-11 81 37T34 159Z"/></g><g data-mml-node="mo" transform="translate(1672.8,0)"><path data-c="3D" d="M56 347Q56 360 70 367H707Q722 359 722 347 722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153 722 140 707 133H70Q56 140 56 153Z"/></g><g data-mml-node="mfrac" transform="translate(2728.6,0)"><g data-mml-node="msub" transform="translate(220,676)"><g data-mml-node="mi"><path data-c="1D441" d="M234 637Q231 637 226 637 201 637 196 638T191 649Q191 676 202 682 204 683 299 683 376 683 387 683T401 677Q612 181 616 168L670 381Q723 592 723 606 723 633 659 637 635 637 635 648 635 650 637 660 641 676 643 679T653 683Q656 683 684 682T767 680Q817 680 843 681T873 682Q888 682 888 672 888 650 880 642 878 637 858 637 787 633 769 597L620 7Q618 0 599 0 585 0 582 2 579 5 453 305L326 604 261 344Q196 88 196 79 201 46 268 46H278Q284 41 284 38T282 19Q278 6 272 0H259Q228 2 151 2 123 2 100 2T63 2 46 1Q31 1 31 10 31 14 34 26T39 40Q41 46 62 46 130 49 150 85 154 91 221 362L289 634Q287 635 234 637Z"/></g><g data-mml-node="TeXAtom" transform="translate(836,-150) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D450" d="M34 159Q34 268 120 355T306 442Q362 442 394 418T427 355Q427 326 408 306T360 285Q341 285 330 295T319 325 330 359 352 380 366 386H367Q367 388 361 392T340 400 306 404Q276 404 249 390 228 381 206 359 162 315 142 235T121 119Q121 73 147 50 169 26 205 26H209Q321 26 394 111 403 121 406 121 410 121 419 112T429 98 420 83 391 55 346 25 282 0 202-11Q127-11 81 37T34 159Z"/></g><g data-mml-node="mi" transform="translate(433,0)"><path data-c="1D45C" d="M201-11Q126-11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441 333 441 341 440 354 437 367 433T402 417 438 387 464 338 476 268Q476 161 390 75T201-11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375 346 405 306 405 243 405 195 347 158 303 140 230T121 120Z"/></g><g data-mml-node="mi" transform="translate(918,0)"><path data-c="1D45F" d="M21 287Q22 290 23 295T28 317 38 348 53 381 73 411 99 433 132 442Q161 442 183 430T214 408 225 388Q227 382 228 382T236 389Q284 441 347 441H350Q398 441 422 400 430 381 430 363 430 333 417 315T391 292 366 288Q346 288 334 299T322 328Q322 376 378 392 356 405 342 405 286 405 239 331 229 315 224 298T190 165Q156 25 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 114 189T154 366Q154 405 128 405 107 405 92 377T68 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g></g></g><g data-mml-node="mi" transform="translate(703,-686)"><path data-c="1D441" d="M234 637Q231 637 226 637 201 637 196 638T191 649Q191 676 202 682 204 683 299 683 376 683 387 683T401 677Q612 181 616 168L670 381Q723 592 723 606 723 633 659 637 635 637 635 648 635 650 637 660 641 676 643 679T653 683Q656 683 684 682T767 680Q817 680 843 681T873 682Q888 682 888 672 888 650 880 642 878 637 858 637 787 633 769 597L620 7Q618 0 599 0 585 0 582 2 579 5 453 305L326 604 261 344Q196 88 196 79 201 46 268 46H278Q284 41 284 38T282 19Q278 6 272 0H259Q228 2 151 2 123 2 100 2T63 2 46 1Q31 1 31 10 31 14 34 26T39 40Q41 46 62 46 130 49 150 85 154 91 221 362L289 634Q287 635 234 637Z"/></g><rect width="2054" height="60" x="120" y="220"/></g></g></g></g></svg><svg data-labels="true" preserveAspectRatio="xMaxYMid" viewBox="1278 -1272.5 1 2045"><g data-labels="true" transform="matrix(1 0 0 -1 0 0)"><g data-mml-node="mtd" id="mjx-eqn:7" transform="translate(0,663.5)"><text data-id-align="true"/><g data-idbox="true" transform="translate(0,-750)"><g data-mml-node="mtext"><path data-c="28" d="M94 250Q94 319 104 381T127 488 164 576 202 643 244 695 277 729 302 750H315 319Q333 750 333 741 333 738 316 720T275 667 226 581 184 443 167 250 184 58 225-81 274-167 316-220 333-241Q333-250 318-250H315 302L274-226Q180-141 137-14T94 250Z"/><path data-c="37" d="M55 458Q56 460 72 567L88 674Q88 676 108 676H128V672Q128 662 143 655T195 646 364 644H485V605L417 512Q408 500 387 472T360 435 339 403 319 367 305 330 292 284 284 230 278 162 275 80Q275 66 275 52T274 28V19Q270 2 255-10T221-22Q210-22 200-19T179 0 168 40Q168 198 265 368 285 400 349 489L395 552H302Q128 552 119 546 113 543 108 522T98 479L95 458V455H55V458Z" transform="translate(389,0)"/><path data-c="29" d="M60 749 64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12 224-76 186-143 145-194 113-227 90-246Q87-249 86-250H74Q66-250 63-250T58-247 55-238Q56-237 66-225 221-64 221 250T66 725Q56 737 55 738 55 746 60 749Z" transform="translate(889,0)"/></g></g></g></g></svg></g></g></g></g></svg></mjx-container></p><p>其中 N 表示文档总数，<mjx-container class="MathJax" jax="SVG"><svg style="vertical-align:-.357ex" xmlns="http://www.w3.org/2000/svg" width="4.195ex" height="1.902ex" role="img" focusable="false" viewBox="0 -683 1854 840.8"><g stroke="currentColor" fill="currentColor" stroke-width="0" transform="scale(1,-1)"><g data-mml-node="math"><g data-mml-node="msub"><g data-mml-node="mi"><path data-c="1D441" d="M234 637Q231 637 226 637 201 637 196 638T191 649Q191 676 202 682 204 683 299 683 376 683 387 683T401 677Q612 181 616 168L670 381Q723 592 723 606 723 633 659 637 635 637 635 648 635 650 637 660 641 676 643 679T653 683Q656 683 684 682T767 680Q817 680 843 681T873 682Q888 682 888 672 888 650 880 642 878 637 858 637 787 633 769 597L620 7Q618 0 599 0 585 0 582 2 579 5 453 305L326 604 261 344Q196 88 196 79 201 46 268 46H278Q284 41 284 38T282 19Q278 6 272 0H259Q228 2 151 2 123 2 100 2T63 2 46 1Q31 1 31 10 31 14 34 26T39 40Q41 46 62 46 130 49 150 85 154 91 221 362L289 634Q287 635 234 637Z"/></g><g data-mml-node="TeXAtom" transform="translate(836,-150) scale(0.707)" data-mjx-texclass="ORD"><g data-mml-node="mi"><path data-c="1D450" d="M34 159Q34 268 120 355T306 442Q362 442 394 418T427 355Q427 326 408 306T360 285Q341 285 330 295T319 325 330 359 352 380 366 386H367Q367 388 361 392T340 400 306 404Q276 404 249 390 228 381 206 359 162 315 142 235T121 119Q121 73 147 50 169 26 205 26H209Q321 26 394 111 403 121 406 121 410 121 419 112T429 98 420 83 391 55 346 25 282 0 202-11Q127-11 81 37T34 159Z"/></g><g data-mml-node="mi" transform="translate(433,0)"><path data-c="1D45C" d="M201-11Q126-11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441 333 441 341 440 354 437 367 433T402 417 438 387 464 338 476 268Q476 161 390 75T201-11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375 346 405 306 405 243 405 195 347 158 303 140 230T121 120Z"/></g><g data-mml-node="mi" transform="translate(918,0)"><path data-c="1D45F" d="M21 287Q22 290 23 295T28 317 38 348 53 381 73 411 99 433 132 442Q161 442 183 430T214 408 225 388Q227 382 228 382T236 389Q284 441 347 441H350Q398 441 422 400 430 381 430 363 430 333 417 315T391 292 366 288Q346 288 334 299T322 328Q322 376 378 392 356 405 342 405 286 405 239 331 229 315 224 298T190 165Q156 25 151 16 138-11 108-11 95-11 87-5T76 7 74 17Q74 30 114 189T154 366Q154 405 128 405 107 405 92 377T68 316 57 280Q55 278 41 278H27Q21 284 21 287Z"/></g></g></g></g></g></svg></mjx-container>表示正确聚类的文档数.</p></div><div class="story post-story"><h2 id="实验过程"><a href="#实验过程" class="headerlink" title="实验过程"></a>实验过程</h2><h3 id="准备数据，上传至HDFS"><a href="#准备数据，上传至HDFS" class="headerlink" title="准备数据，上传至HDFS"></a>准备数据，上传至 HDFS</h3><p>HDFS 创建文件夹</p><p><img src="https://static.mhuig.top/npm/imbox@0.0.3/data/202082162416.png" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.3/data/202082162416.png" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="HDFS创建文件夹"></p><p>hadoop 关闭安全模式</p><p><img src="https://static.mhuig.top/npm/imbox@0.0.3/data/20208216240.png" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.3/data/20208216240.png" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="hadoop关闭安全模式"></p><p>上传 KDD Cup 1999 数据集</p><p><img src="https://static.mhuig.top/npm/imbox@0.0.3/data/202082162343.png" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.3/data/202082162343.png" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="上传KDD Cup 1999 数据集"></p><p><img src="https://static.mhuig.top/npm/imbox@0.0.3/data/202082162318.png" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.3/data/202082162318.png" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="上传KDD Cup 1999 数据集"></p><p><img src="https://static.mhuig.top/npm/imbox@0.0.3/data/202082162254.png" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.3/data/202082162254.png" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="上传KDD Cup 1999 数据集"></p><p>查看上传成功</p><p><img src="https://static.mhuig.top/npm/imbox@0.0.3/data/202082162233.png" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.3/data/202082162233.png" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="查看上传成功"></p><h3 id="通过kddcup-names加载列名称"><a href="#通过kddcup-names加载列名称" class="headerlink" title="通过kddcup.names加载列名称"></a>通过 kddcup.names 加载列名称</h3><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">names=[]</span><br><span class="line"><span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">"/export/work/F/3/data/kddcup.names"</span>) <span class="keyword">as</span> f:</span><br><span class="line"> line = f.readline()</span><br><span class="line"> line = f.readline()</span><br><span class="line"> <span class="keyword">while</span> line:</span><br><span class="line"> names.append(line.split(<span class="string">":"</span>)[<span class="number">0</span>])</span><br><span class="line"> line = f.readline()</span><br><span class="line"></span><br><span class="line">names.append(<span class="string">"label"</span>)</span><br></pre></td></tr></tbody></table></figure><p>输出列名称：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">[<span class="string">'duration'</span>, <span class="string">'protocol_type'</span>, <span class="string">'service'</span>, <span class="string">'flag'</span>, <span class="string">'src_bytes'</span>, <span class="string">'dst_bytes'</span>, <span class="string">'land'</span>, <span class="string">'wrong_fragment'</span>, <span class="string">'urgent'</span>, <span class="string">'hot'</span>, <span class="string">'num_failed_logins'</span>, <span class="string">'logged_in'</span>, <span class="string">'num_compromised'</span>, <span class="string">'root_shell'</span>, <span class="string">'su_attempted'</span>, <span class="string">'num_root'</span>, <span class="string">'num_file_creations'</span>, <span class="string">'num_shells'</span>, <span class="string">'num_access_files'</span>, <span class="string">'num_outbound_cmds'</span>, <span class="string">'is_host_login'</span>, <span class="string">'is_guest_login'</span>, <span class="string">'count'</span>, <span class="string">'srv_count'</span>, <span class="string">'serror_rate'</span>, <span class="string">'srv_serror_rate'</span>, <span class="string">'rerror_rate'</span>, <span class="string">'srv_rerror_rate'</span>, <span class="string">'same_srv_rate'</span>, <span class="string">'diff_srv_rate'</span>, <span class="string">'srv_diff_host_rate'</span>, <span class="string">'dst_host_count'</span>, <span class="string">'dst_host_srv_count'</span>, <span class="string">'dst_host_same_srv_rate'</span>, <span class="string">'dst_host_diff_srv_rate'</span>, <span class="string">'dst_host_same_src_port_rate'</span>, <span class="string">'dst_host_srv_diff_host_rate'</span>, <span class="string">'dst_host_serror_rate'</span>, <span class="string">'dst_host_srv_serror_rate'</span>, <span class="string">'dst_host_rerror_rate'</span>, <span class="string">'dst_host_srv_rerror_rate'</span>, <span class="string">'label'</span>]</span><br></pre></td></tr></tbody></table></figure><h3 id="构建Dataframe"><a href="#构建Dataframe" class="headerlink" title="构建Dataframe"></a>构建 Dataframe</h3><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">names=[<span class="string">'duration'</span>, <span class="string">'protocol_type'</span>, <span class="string">'service'</span>, <span class="string">'flag'</span>, <span class="string">'src_bytes'</span>, <span class="string">'dst_bytes'</span>, <span class="string">'land'</span>, <span class="string">'wrong_fragment'</span>, <span class="string">'urgent'</span>, <span class="string">'hot'</span>, <span class="string">'num_failed_logins'</span>, <span class="string">'logged_in'</span>, <span class="string">'num_compromised'</span>, <span class="string">'root_shell'</span>, <span class="string">'su_attempted'</span>, <span class="string">'num_root'</span>, <span class="string">'num_file_creations'</span>, <span class="string">'num_shells'</span>, <span class="string">'num_access_files'</span>, <span class="string">'num_outbound_cmds'</span>, <span class="string">'is_host_login'</span>, <span class="string">'is_guest_login'</span>, <span class="string">'count'</span>, <span class="string">'srv_count'</span>, <span class="string">'serror_rate'</span>, <span class="string">'srv_serror_rate'</span>, <span class="string">'rerror_rate'</span>, <span class="string">'srv_rerror_rate'</span>, <span class="string">'same_srv_rate'</span>, <span class="string">'diff_srv_rate'</span>, <span class="string">'srv_diff_host_rate'</span>, <span class="string">'dst_host_count'</span>, <span class="string">'dst_host_srv_count'</span>, <span class="string">'dst_host_same_srv_rate'</span>, <span class="string">'dst_host_diff_srv_rate'</span>, <span class="string">'dst_host_same_src_port_rate'</span>, <span class="string">'dst_host_srv_diff_host_rate'</span>, <span class="string">'dst_host_serror_rate'</span>, <span class="string">'dst_host_srv_serror_rate'</span>, <span class="string">'dst_host_rerror_rate'</span>, <span class="string">'dst_host_srv_rerror_rate'</span>, <span class="string">'label'</span>]</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> pyspark.sql.types <span class="keyword">import</span> Row</span><br><span class="line"><span class="keyword">from</span> pyspark.sql.types <span class="keyword">import</span> StructType</span><br><span class="line"><span class="keyword">from</span> pyspark.sql.types <span class="keyword">import</span> StructField</span><br><span class="line"><span class="keyword">from</span> pyspark.sql.types <span class="keyword">import</span> StringType</span><br><span class="line"><span class="keyword">from</span> pyspark.sql.types <span class="keyword">import</span> FloatType</span><br><span class="line"><span class="keyword">from</span> pyspark.conf <span class="keyword">import</span> SparkConf</span><br><span class="line"><span class="keyword">from</span> pyspark <span class="keyword">import</span> SparkContext</span><br><span class="line"><span class="keyword">from</span> pyspark.sql.session <span class="keyword">import</span> SparkSession</span><br><span class="line"><span class="comment"># 构建Dataframe</span></span><br><span class="line">conf = SparkConf().setAppName(<span class="string">"applicaiton"</span>).<span class="built_in">set</span>(<span class="string">"spark.executor.heartbeatInterval"</span>,<span class="string">"200000"</span>).<span class="built_in">set</span>(<span class="string">"spark.network.timeout"</span>,<span class="string">"300000"</span>)</span><br><span class="line">sc = SparkContext.getOrCreate(conf)</span><br><span class="line">spark = SparkSession(sc)</span><br><span class="line">testRDD = sc.textFile(<span class="string">"/3/corrected"</span>)</span><br><span class="line">fields = <span class="built_in">list</span>(<span class="built_in">map</span>( <span class="keyword">lambda</span> fieldName : StructField(fieldName, StringType(), nullable = <span class="literal">True</span>) <span class="keyword">if</span> fieldName <span class="keyword">in</span> [<span class="string">"protocol_type"</span>, <span class="string">"service"</span>, <span class="string">"flag"</span>,<span class="string">"label"</span>] <span class="keyword">else</span> StructField(fieldName, FloatType(), nullable = <span class="literal">True</span>) , names))</span><br><span class="line">schema = StructType(fields)</span><br><span class="line">rowRDD = testRDD.<span class="built_in">map</span>(<span class="keyword">lambda</span> line : line.split(<span class="string">","</span>)).<span class="built_in">map</span>(<span class="keyword">lambda</span> attr : Row(<span class="built_in">float</span>(attr[<span class="number">0</span>]),attr[<span class="number">1</span>],attr[<span class="number">2</span>],attr[<span class="number">3</span>],<span class="built_in">float</span>(attr[<span class="number">4</span>]),<span class="built_in">float</span>(attr[<span class="number">5</span>]),<span class="built_in">float</span>(attr[<span class="number">6</span>]),<span class="built_in">float</span>(attr[<span class="number">7</span>]),<span class="built_in">float</span>(attr[<span class="number">8</span>]),<span class="built_in">float</span>(attr[<span class="number">9</span>]),<span class="built_in">float</span>(attr[<span class="number">10</span>]),<span class="built_in">float</span>(attr[<span class="number">11</span>]),<span class="built_in">float</span>(attr[<span class="number">12</span>]),<span class="built_in">float</span>(attr[<span class="number">13</span>]),<span class="built_in">float</span>(attr[<span class="number">14</span>]),<span class="built_in">float</span>(attr[<span class="number">15</span>]),<span class="built_in">float</span>(attr[<span class="number">16</span>]),<span class="built_in">float</span>(attr[<span class="number">17</span>]),<span class="built_in">float</span>(attr[<span class="number">18</span>]),<span class="built_in">float</span>(attr[<span class="number">19</span>]),<span class="built_in">float</span>(attr[<span class="number">20</span>]),<span class="built_in">float</span>(attr[<span class="number">21</span>]),<span class="built_in">float</span>(attr[<span class="number">22</span>]),<span class="built_in">float</span>(attr[<span class="number">23</span>]),<span class="built_in">float</span>(attr[<span class="number">24</span>]),<span class="built_in">float</span>(attr[<span class="number">25</span>]),<span class="built_in">float</span>(attr[<span class="number">26</span>]),<span class="built_in">float</span>(attr[<span class="number">27</span>]),<span class="built_in">float</span>(attr[<span class="number">28</span>]),<span class="built_in">float</span>(attr[<span class="number">29</span>]),<span class="built_in">float</span>(attr[<span class="number">30</span>]),<span class="built_in">float</span>(attr[<span class="number">31</span>]),<span class="built_in">float</span>(attr[<span class="number">32</span>]),<span class="built_in">float</span>(attr[<span class="number">33</span>]),<span class="built_in">float</span>(attr[<span class="number">34</span>]),<span class="built_in">float</span>(attr[<span class="number">35</span>]),<span class="built_in">float</span>(attr[<span class="number">36</span>]),<span class="built_in">float</span>(attr[<span class="number">37</span>]),<span class="built_in">float</span>(attr[<span class="number">38</span>]),<span class="built_in">float</span>(attr[<span class="number">39</span>]),<span class="built_in">float</span>(attr[<span class="number">40</span>]),attr[<span class="number">41</span>]))</span><br><span class="line">testDF = spark.createDataFrame(rowRDD, schema)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">dataRDD = sc.textFile(<span class="string">"/3/kddcup.data"</span>)</span><br><span class="line">fields = <span class="built_in">list</span>(<span class="built_in">map</span>( <span class="keyword">lambda</span> fieldName : StructField(fieldName, StringType(), nullable = <span class="literal">True</span>) <span class="keyword">if</span> fieldName <span class="keyword">in</span> [<span class="string">"protocol_type"</span>, <span class="string">"service"</span>, <span class="string">"flag"</span>,<span class="string">"label"</span>] <span class="keyword">else</span> StructField(fieldName, FloatType(), nullable = <span class="literal">True</span>) , names))</span><br><span class="line">schema = StructType(fields)</span><br><span class="line">rowRDD = dataRDD.<span class="built_in">map</span>(<span class="keyword">lambda</span> line : line.split(<span class="string">","</span>)).<span class="built_in">map</span>(<span class="keyword">lambda</span> attr : Row(<span class="built_in">float</span>(attr[<span class="number">0</span>]),attr[<span class="number">1</span>],attr[<span class="number">2</span>],attr[<span class="number">3</span>],<span class="built_in">float</span>(attr[<span class="number">4</span>]),<span class="built_in">float</span>(attr[<span class="number">5</span>]),<span class="built_in">float</span>(attr[<span class="number">6</span>]),<span class="built_in">float</span>(attr[<span class="number">7</span>]),<span class="built_in">float</span>(attr[<span class="number">8</span>]),<span class="built_in">float</span>(attr[<span class="number">9</span>]),<span class="built_in">float</span>(attr[<span class="number">10</span>]),<span class="built_in">float</span>(attr[<span class="number">11</span>]),<span class="built_in">float</span>(attr[<span class="number">12</span>]),<span class="built_in">float</span>(attr[<span class="number">13</span>]),<span class="built_in">float</span>(attr[<span class="number">14</span>]),<span class="built_in">float</span>(attr[<span class="number">15</span>]),<span class="built_in">float</span>(attr[<span class="number">16</span>]),<span class="built_in">float</span>(attr[<span class="number">17</span>]),<span class="built_in">float</span>(attr[<span class="number">18</span>]),<span class="built_in">float</span>(attr[<span class="number">19</span>]),<span class="built_in">float</span>(attr[<span class="number">20</span>]),<span class="built_in">float</span>(attr[<span class="number">21</span>]),<span class="built_in">float</span>(attr[<span class="number">22</span>]),<span class="built_in">float</span>(attr[<span class="number">23</span>]),<span class="built_in">float</span>(attr[<span class="number">24</span>]),<span class="built_in">float</span>(attr[<span class="number">25</span>]),<span class="built_in">float</span>(attr[<span class="number">26</span>]),<span class="built_in">float</span>(attr[<span class="number">27</span>]),<span class="built_in">float</span>(attr[<span class="number">28</span>]),<span class="built_in">float</span>(attr[<span class="number">29</span>]),<span class="built_in">float</span>(attr[<span class="number">30</span>]),<span class="built_in">float</span>(attr[<span class="number">31</span>]),<span class="built_in">float</span>(attr[<span class="number">32</span>]),<span class="built_in">float</span>(attr[<span class="number">33</span>]),<span class="built_in">float</span>(attr[<span class="number">34</span>]),<span class="built_in">float</span>(attr[<span class="number">35</span>]),<span class="built_in">float</span>(attr[<span class="number">36</span>]),<span class="built_in">float</span>(attr[<span class="number">37</span>]),<span class="built_in">float</span>(attr[<span class="number">38</span>]),<span class="built_in">float</span>(attr[<span class="number">39</span>]),<span class="built_in">float</span>(attr[<span class="number">40</span>]),attr[<span class="number">41</span>]))</span><br><span class="line">dataDF = spark.createDataFrame(rowRDD, schema)</span><br></pre></td></tr></tbody></table></figure><h3 id="数据集统计"><a href="#数据集统计" class="headerlink" title="数据集统计"></a>数据集统计</h3><p>统计数据集中各个类别标号以及每类样本有多少，并展示。</p><p>数据集的类别标号以及每类样本数</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">dataDF.groupBy(<span class="string">"label"</span>).count().show(<span class="number">10000</span>)</span><br><span class="line"></span><br><span class="line">+----------------+-------+ </span><br><span class="line">| label| count|</span><br><span class="line">+----------------+-------+</span><br><span class="line">| warezmaster.| <span class="number">20</span>|</span><br><span class="line">| smurf.|<span class="number">2807886</span>|</span><br><span class="line">| pod.| <span class="number">264</span>|</span><br><span class="line">| imap.| <span class="number">12</span>|</span><br><span class="line">| nmap.| <span class="number">2316</span>|</span><br><span class="line">| guess_passwd.| <span class="number">53</span>|</span><br><span class="line">| ipsweep.| <span class="number">12481</span>|</span><br><span class="line">| portsweep.| <span class="number">10413</span>|</span><br><span class="line">| satan.| <span class="number">15892</span>|</span><br><span class="line">| land.| <span class="number">21</span>|</span><br><span class="line">| loadmodule.| <span class="number">9</span>|</span><br><span class="line">| ftp_write.| <span class="number">8</span>|</span><br><span class="line">|buffer_overflow.| <span class="number">30</span>|</span><br><span class="line">| rootkit.| <span class="number">10</span>|</span><br><span class="line">| warezclient.| <span class="number">1020</span>|</span><br><span class="line">| teardrop.| <span class="number">979</span>|</span><br><span class="line">| perl.| <span class="number">3</span>|</span><br><span class="line">| phf.| <span class="number">4</span>|</span><br><span class="line">| multihop.| <span class="number">7</span>|</span><br><span class="line">| neptune.|<span class="number">1072017</span>|</span><br><span class="line">| back.| <span class="number">2203</span>|</span><br><span class="line">| spy.| <span class="number">2</span>|</span><br><span class="line">| normal.| <span class="number">972781</span>|</span><br><span class="line">+----------------+-------+</span><br></pre></td></tr></tbody></table></figure><p>测试集的类别标号以及每类样本数</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">testDF.groupBy(<span class="string">"label"</span>).count().show(<span class="number">10000</span>) </span><br><span class="line"></span><br><span class="line">+----------------+------+</span><br><span class="line">| label| count|</span><br><span class="line">+----------------+------+</span><br><span class="line">| snmpguess.| <span class="number">2406</span>|</span><br><span class="line">| xlock.| <span class="number">9</span>|</span><br><span class="line">| warezmaster.| <span class="number">1602</span>|</span><br><span class="line">| processtable.| <span class="number">759</span>|</span><br><span class="line">| smurf.|<span class="number">164091</span>|</span><br><span class="line">| pod.| <span class="number">87</span>|</span><br><span class="line">| worm.| <span class="number">2</span>|</span><br><span class="line">| snmpgetattack.| <span class="number">7741</span>|</span><br><span class="line">| mscan.| <span class="number">1053</span>|</span><br><span class="line">| nmap.| <span class="number">84</span>|</span><br><span class="line">| imap.| <span class="number">1</span>|</span><br><span class="line">| xterm.| <span class="number">13</span>|</span><br><span class="line">| sqlattack.| <span class="number">2</span>|</span><br><span class="line">| guess_passwd.| <span class="number">4367</span>|</span><br><span class="line">| mailbomb.| <span class="number">5000</span>|</span><br><span class="line">| xsnoop.| <span class="number">4</span>|</span><br><span class="line">| ipsweep.| <span class="number">306</span>|</span><br><span class="line">| portsweep.| <span class="number">354</span>|</span><br><span class="line">| named.| <span class="number">17</span>|</span><br><span class="line">| satan.| <span class="number">1633</span>|</span><br><span class="line">| land.| <span class="number">9</span>|</span><br><span class="line">| loadmodule.| <span class="number">2</span>|</span><br><span class="line">| ftp_write.| <span class="number">3</span>|</span><br><span class="line">| sendmail.| <span class="number">17</span>|</span><br><span class="line">|buffer_overflow.| <span class="number">22</span>|</span><br><span class="line">| httptunnel.| <span class="number">158</span>|</span><br><span class="line">| apache2.| <span class="number">794</span>|</span><br><span class="line">| saint.| <span class="number">736</span>|</span><br><span class="line">| rootkit.| <span class="number">13</span>|</span><br><span class="line">| teardrop.| <span class="number">12</span>|</span><br><span class="line">| perl.| <span class="number">2</span>|</span><br><span class="line">| phf.| <span class="number">2</span>|</span><br><span class="line">| multihop.| <span class="number">18</span>|</span><br><span class="line">| udpstorm.| <span class="number">2</span>|</span><br><span class="line">| neptune.| <span class="number">58001</span>|</span><br><span class="line">| back.| <span class="number">1098</span>|</span><br><span class="line">| ps.| <span class="number">16</span>|</span><br><span class="line">| normal.| <span class="number">60593</span>|</span><br><span class="line">+----------------+------+</span><br></pre></td></tr></tbody></table></figure><h3 id="尝试聚类"><a href="#尝试聚类" class="headerlink" title="尝试聚类"></a>尝试聚类</h3><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> pyspark.ml <span class="keyword">import</span> Pipeline,PipelineModel</span><br><span class="line"><span class="keyword">from</span> pyspark.ml.clustering <span class="keyword">import</span> KMeans,KMeansModel</span><br><span class="line"><span class="keyword">from</span> pyspark.ml.feature <span class="keyword">import</span> VectorAssembler</span><br><span class="line"><span class="keyword">from</span> pyspark.sql <span class="keyword">import</span> DataFrame</span><br><span class="line"><span class="keyword">import</span> random</span><br></pre></td></tr></tbody></table></figure><p>用 VectorAssembler 创建一个特征向量，基于这些特征向量用一个 K 均值实现来创建一个模型，再用一个管道将它们拼接在一起。从得到的模型中，可以提取并检验簇群中心。</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">numericOnly = dataDF.drop(<span class="string">"protocol_type"</span>, <span class="string">"service"</span>, <span class="string">"flag"</span>).cache()</span><br><span class="line">assembler = VectorAssembler(inputCols=numericOnly.drop(<span class="string">"label"</span>).columns, outputCol=<span class="string">"featureVector"</span>)</span><br><span class="line">kmeans = KMeans().setPredictionCol(<span class="string">"cluster"</span>).setFeaturesCol(<span class="string">"featureVector"</span>)</span><br><span class="line">pipeline = Pipeline().setStages([assembler, kmeans])</span><br><span class="line">pipelineModel = pipeline.fit(numericOnly)</span><br><span class="line">kmeansModel = pipelineModel.stages[-<span class="number">1</span>]</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> kmeansModel.clusterCenters():<span class="built_in">print</span>(i)</span><br></pre></td></tr></tbody></table></figure><p>输出：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">[<span class="number">4.83401949e+01</span> <span class="number">1.83462154e+03</span> <span class="number">8.26203195e+02</span> <span class="number">5.71611720e-06</span> </span><br><span class="line"> <span class="number">6.48779303e-04</span> <span class="number">7.96173468e-06</span> <span class="number">1.24376586e-02</span> <span class="number">3.20510858e-05</span></span><br><span class="line"> <span class="number">1.43529049e-01</span> <span class="number">8.08830584e-03</span> <span class="number">6.81851124e-05</span> <span class="number">3.67464677e-05</span></span><br><span class="line"> <span class="number">1.29349608e-02</span> <span class="number">1.18874823e-03</span> <span class="number">7.43095237e-05</span> <span class="number">1.02114351e-03</span></span><br><span class="line"> <span class="number">0.00000000e+00</span> <span class="number">4.08294086e-07</span> <span class="number">8.35165553e-04</span> <span class="number">3.34973508e+02</span></span><br><span class="line"> <span class="number">2.95267146e+02</span> <span class="number">1.77970317e-01</span> <span class="number">1.78036989e-01</span> <span class="number">5.76648988e-02</span></span><br><span class="line"> <span class="number">5.77299094e-02</span> <span class="number">7.89884132e-01</span> <span class="number">2.11796105e-02</span> <span class="number">2.82608102e-02</span></span><br><span class="line"> <span class="number">2.32981078e+02</span> <span class="number">1.89214283e+02</span> <span class="number">7.53713390e-01</span> <span class="number">3.07109788e-02</span></span><br><span class="line"> <span class="number">6.05051931e-01</span> <span class="number">6.46410786e-03</span> <span class="number">1.78091184e-01</span> <span class="number">1.77885898e-01</span></span><br><span class="line"> <span class="number">5.79276115e-02</span> <span class="number">5.76592214e-02</span>]</span><br><span class="line">[<span class="number">1.09990000e+04</span> <span class="number">0.00000000e+00</span> <span class="number">1.30993741e+09</span> <span class="number">0.00000000e+00</span></span><br><span class="line"> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span></span><br><span class="line"> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span></span><br><span class="line"> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span></span><br><span class="line"> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">1.00000000e+00</span></span><br><span class="line"> <span class="number">1.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">1.00000000e+00</span></span><br><span class="line"> <span class="number">1.00000000e+00</span> <span class="number">1.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span></span><br><span class="line"> <span class="number">2.55000000e+02</span> <span class="number">1.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">6.49999976e-01</span></span><br><span class="line"> <span class="number">1.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span> <span class="number">0.00000000e+00</span></span><br><span class="line"> <span class="number">1.00000000e+00</span> <span class="number">1.00000000e+00</span>]</span><br></pre></td></tr></tbody></table></figure><p>对这些数字做一个直观的解释并不容易，但是每一个数字都表示模型生成的一个簇群中心，也称为质心（centroid）。就每个数值输入特征而言，这些值是质心的坐标。</p><h3 id="k的选择"><a href="#k的选择" class="headerlink" title="k的选择"></a>k 的选择</h3><p>如果每个数据点都紧靠最近的质心，则可认为聚类是较优的。这里的 “近” 采用欧氏距离定义。这是评估聚类质量的一种简单又常用的方法，使用与所有点之间距离的平均值，有时也可以使用平方距离的平均值。实际上，KMeansModel 提供了一个 computeCost 方法来计算平方距离的总和，并且很容易用来计算平方距离的平均值。</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">numericOnly = dataDF.drop(<span class="string">"protocol_type"</span>, <span class="string">"service"</span>, <span class="string">"flag"</span>).cache()</span><br><span class="line"><span class="comment"># computeCost 方法来计算平方距离的总和，并且很容易用来计算平方距离的平均值。</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">clusteringScore0</span>(<span class="params">data,k</span>):</span><br><span class="line"> assembler = VectorAssembler(inputCols=data.drop(<span class="string">"label"</span>).columns, outputCol=<span class="string">"featureVector"</span>)</span><br><span class="line"> kmeans = KMeans().setSeed(<span class="built_in">int</span>(random.random()*<span class="number">10</span>)).setK(k).setPredictionCol(<span class="string">"cluster"</span>).setFeaturesCol(<span class="string">"featureVector"</span>) <span class="comment">#.setMaxIter(40).setTol(1.0e-5)</span></span><br><span class="line"> pipeline = Pipeline().setStages([assembler, kmeans])</span><br><span class="line"> pipelineModel = pipeline.fit(data)</span><br><span class="line"> kmeansModel = pipelineModel.stages[-<span class="number">1</span>]</span><br><span class="line"> Srore=kmeansModel.computeCost(assembler.transform(data)) / data.count()</span><br><span class="line"> <span class="keyword">return</span> Srore</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">20</span>, <span class="number">100</span>, <span class="number">20</span>):</span><br><span class="line"> <span class="built_in">print</span>([k, clusteringScore0(numericOnly, k)])</span><br></pre></td></tr></tbody></table></figure><p>输出结果：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">[<span class="number">20</span>, <span class="number">148277112.23861197</span>] </span><br><span class="line">[<span class="number">40</span>, <span class="number">49940659.143821806</span>]</span><br><span class="line">[<span class="number">60</span>, <span class="number">18265796.561388526</span>]</span><br><span class="line">[<span class="number">80</span>, <span class="number">15313289.324247833</span>]</span><br></pre></td></tr></tbody></table></figure><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651998844488/202082162158.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651998844488/202082162158.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="输出结果显示得分随着 k 的增加而降低"></p><p>输出结果显示得分<strong>随着 k 的增加而降低</strong>。</p><p>增加迭代时间可以优化聚类结果。算法提供了 setTol () 来设置一个阈值，该阈值控制聚类过程中簇质心进行有效移动的最小值。降低该阈值能使质心继续移动更长的时间。使用 setMaxIter () 增加最大迭代次数也可以防止它过早停止，代价是可能需要更多的计算。</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">clusteringScore1</span>(<span class="params">data,k</span>):</span><br><span class="line"> assembler = VectorAssembler(inputCols=data.drop(<span class="string">"label"</span>).columns, outputCol=<span class="string">"featureVector"</span>)</span><br><span class="line"> kmeans = KMeans().setSeed(<span class="built_in">int</span>(random.random()*<span class="number">10</span>)).setK(k).setPredictionCol(<span class="string">"cluster"</span>).setFeaturesCol(<span class="string">"featureVector"</span>).setMaxIter(<span class="number">40</span>).setTol(<span class="number">1.0e-5</span>)</span><br><span class="line"> pipeline = Pipeline().setStages([assembler, kmeans])</span><br><span class="line"> pipelineModel = pipeline.fit(data)</span><br><span class="line"> kmeansModel = pipelineModel.stages[-<span class="number">1</span>]</span><br><span class="line"> Srore=kmeansModel.computeCost(assembler.transform(data)) / data.count()</span><br><span class="line"> <span class="keyword">return</span> Srore</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">20</span>, <span class="number">120</span>, <span class="number">20</span>):</span><br><span class="line"> <span class="built_in">print</span>([k, clusteringScore1(numericOnly, k)])</span><br></pre></td></tr></tbody></table></figure><p>输出结果：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">[<span class="number">20</span>, <span class="number">148277112.23861197</span>] </span><br><span class="line">[<span class="number">40</span>, <span class="number">11564470.915401561</span>]</span><br><span class="line">[<span class="number">60</span>, <span class="number">16343181.409780543</span>]</span><br><span class="line">[<span class="number">80</span>, <span class="number">22323383.079484705</span>]</span><br><span class="line">[<span class="number">100</span>, <span class="number">7572838.84573523</span>]</span><br></pre></td></tr></tbody></table></figure><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651998932373/202082162140.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651998932373/202082162140.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="在 k 过了 100 这个点之后得分下降还是很明显，所以 k 的拐点值应该大于 100。"></p><p>糟糕的情况是，前面的结果中 k=80 时的距离居然比 k=60 的距离大。这不应该发生，因为 k 取更大值时，聚类的结果应该至少与 k 取一个较小值时的结果一样好。问题的原因在于，这种给定 k 值的 K 均值算法并不一定能得到最优聚类。K 均值的迭代过程是从一个随机点开始的，因此可能收敛于一个局部最小值，这个局部最小值可能还不错，但并不是全局最优的。</p><p><strong>在 k 过了 100 这个点之后得分下降还是很明显，所以 k 的拐点值应该大于 100。</strong></p><h3 id="特征的规范化-1"><a href="#特征的规范化-1" class="headerlink" title="特征的规范化"></a>特征的规范化</h3><p>特征的规范化可以通过将每个特征转换为标准得分来完成。这就是说用对每个特征值求平均，用每个特征值减去平均值，然后除以特征值的标准差。</p><p>由于减去平均值相当于把所有数据点沿相同方法移动相同距离，不影响点之间的欧氏距离，所以实际上减去平均值对聚类结果没有影响。</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> pyspark.ml.feature <span class="keyword">import</span> StandardScaler</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">clusteringScore2</span>(<span class="params">data,k</span>):</span><br><span class="line"> assembler = VectorAssembler(inputCols=data.drop(<span class="string">"label"</span>).columns, outputCol=<span class="string">"featureVector"</span>)</span><br><span class="line"> scaler = StandardScaler(inputCol=<span class="string">"featureVector"</span>, outputCol=<span class="string">"scaledFeatureVector"</span>, withStd=<span class="literal">True</span>, withMean=<span class="literal">False</span>)</span><br><span class="line"> kmeans = KMeans().setSeed(<span class="built_in">int</span>(random.random()*<span class="number">10</span>)).setK(k).setPredictionCol(<span class="string">"cluster"</span>).setFeaturesCol(<span class="string">"scaledFeatureVector"</span>).setMaxIter(<span class="number">40</span>).setTol(<span class="number">1.0e-5</span>)</span><br><span class="line"> pipeline = Pipeline().setStages([assembler,scaler,kmeans])</span><br><span class="line"> pipelineModel = pipeline.fit(data)</span><br><span class="line"> kmeansModel = pipelineModel.stages[-<span class="number">1</span>]</span><br><span class="line"> Srore=kmeansModel.computeCost(pipelineModel.transform(data)) / data.count()</span><br><span class="line"> <span class="keyword">return</span> Srore</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">60</span>, <span class="number">300</span>, <span class="number">30</span>):</span><br><span class="line"> <span class="built_in">print</span>([k, clusteringScore2(numericOnly, k)])</span><br></pre></td></tr></tbody></table></figure><p>这有助于将维度放到更平等的基准上，而且在绝对的意义上，看点之间的绝对距离（也就是代价）要小得多。然而，k 值还没有出现一个明显的点，超过该点后，增加 k 值对于改善代价没有明显的作用：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">[<span class="number">60</span>, <span class="number">1.1611941370693641</span>]</span><br><span class="line">[<span class="number">90</span>, <span class="number">0.7236962692254361</span>]</span><br><span class="line">[<span class="number">120</span>, <span class="number">0.5581874996147724</span>]</span><br><span class="line">[<span class="number">150</span>, <span class="number">0.3886887438817504</span>]</span><br><span class="line">[<span class="number">180</span>, <span class="number">0.3333248112741165</span>]</span><br><span class="line">[<span class="number">210</span>, <span class="number">0.27497680552057235</span>]</span><br><span class="line">[<span class="number">240</span>, <span class="number">0.2556693718314817</span>]</span><br><span class="line">[<span class="number">270</span>, <span class="number">0.22710138015576076</span>]</span><br></pre></td></tr></tbody></table></figure><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651998980409/202082162121.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651998980409/202082162121.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="特征的规范化"></p><h3 id="类别型变量-1"><a href="#类别型变量-1" class="headerlink" title="类别型变量"></a>类别型变量</h3><p>归一化使聚类结果有了可贵的进步，但聚类结果还有进一步提升的空间。比如说，几个特征由于不是数值型就被去掉了，于是这些特征里有价值的信息也被丢掉了。如果将这些信息以某种形式加回来，我们应该能得到更好的聚类。</p><p>类别型特征可以用 one-hot 编码转换为几个二元特征，这几个二元特征可以看成数值型维度。举个例子，数据集的第二列代表协议类型，取值可能是 tcp、udp 或 icmp。可以把它们看成 3 个特征，分别取名为 is_tcp、is_udp 和 is_icmp。这样，特征值 tcp 就变成 1,0,0，udp 对应 0,1,0，icmp 对应 0,0,1，以此类推。</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">from</span> pyspark.ml.feature <span class="keyword">import</span> OneHotEncoder, StringIndexer</span><br><span class="line"><span class="comment"># 类别型特征可以用 one-hot 编码转换为几个二元特征，这几个二元特征可以看成数值型维度。</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">oneHotPipeline</span>(<span class="params">inputCol</span>):</span><br><span class="line"> indexer = StringIndexer(inputCol=inputCol,outputCol=inputCol + <span class="string">"_indexed"</span>).setHandleInvalid(<span class="string">"keep"</span>)</span><br><span class="line"> encoder = OneHotEncoder(inputCol=inputCol + <span class="string">"_indexed"</span>,outputCol=inputCol + <span class="string">"_vec"</span>)</span><br><span class="line"> pipeline = Pipeline().setStages([indexer, encoder])</span><br><span class="line"> <span class="keyword">return</span> (pipeline, inputCol + <span class="string">"_vec"</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">clusteringScore3</span>(<span class="params">data,k</span>):</span><br><span class="line"> protoTypeEncoder, protoTypeVecCol = oneHotPipeline(<span class="string">"protocol_type"</span>)</span><br><span class="line"> serviceEncoder, serviceVecCol = oneHotPipeline(<span class="string">"service"</span>)</span><br><span class="line"> flagEncoder, flagVecCol = oneHotPipeline(<span class="string">"flag"</span>)</span><br><span class="line"> assembleCols = (<span class="built_in">set</span>(data.columns)-<span class="built_in">set</span>([<span class="string">"label"</span>, <span class="string">"protocol_type"</span>, <span class="string">"service"</span>, <span class="string">"flag"</span>])).union(<span class="built_in">set</span>([protoTypeVecCol, serviceVecCol, flagVecCol]))</span><br><span class="line"> assembler = VectorAssembler(inputCols=<span class="built_in">list</span>(assembleCols), outputCol=<span class="string">"featureVector"</span>)</span><br><span class="line"> scaler = StandardScaler(inputCol=<span class="string">"featureVector"</span>, outputCol=<span class="string">"scaledFeatureVector"</span>, withStd=<span class="literal">True</span>, withMean=<span class="literal">False</span>)</span><br><span class="line"> kmeans = KMeans().setSeed(<span class="built_in">int</span>(random.random()*<span class="number">10</span>)).setK(k).setPredictionCol(<span class="string">"cluster"</span>).setFeaturesCol(<span class="string">"scaledFeatureVector"</span>).setMaxIter(<span class="number">40</span>).setTol(<span class="number">1.0e-5</span>)</span><br><span class="line"> pipeline = Pipeline().setStages([protoTypeEncoder, serviceEncoder, flagEncoder, assembler, scaler, kmeans])</span><br><span class="line"> pipelineModel = pipeline.fit(data)</span><br><span class="line"> kmeansModel = pipelineModel.stages[-<span class="number">1</span>]</span><br><span class="line"> Srore=kmeansModel.computeCost(pipelineModel.transform(data)) / data.count()</span><br><span class="line"> <span class="keyword">return</span> Srore</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">60</span>, <span class="number">300</span>, <span class="number">30</span>):</span><br><span class="line"> <span class="built_in">print</span>([k, clusteringScore3(dataDF, k)])</span><br></pre></td></tr></tbody></table></figure><p>输出：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">[<span class="number">60</span>, <span class="number">38.01382297522162</span>]</span><br><span class="line">[<span class="number">90</span>, <span class="number">16.419330083446177</span>]</span><br><span class="line">[<span class="number">120</span>, <span class="number">3.2093992442174235</span>]</span><br><span class="line">[<span class="number">150</span>, <span class="number">2.1454678299121843</span>]</span><br><span class="line">[<span class="number">180</span>, <span class="number">1.6142523558430413</span>]</span><br><span class="line">[<span class="number">210</span>, <span class="number">1.3533093788147306</span>]</span><br><span class="line">[<span class="number">240</span>, <span class="number">1.0616778921723296</span>]</span><br><span class="line">[<span class="number">270</span>, <span class="number">0.9068134376554267</span>]</span><br></pre></td></tr></tbody></table></figure><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651999052076/20208216211.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651999052076/20208216211.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="类别型变量"></p><p>局部放大：</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651999083338/202082162045.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651999083338/202082162045.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="局部放大"></p><p>这些样本结果表明，从 k=180 这个点开始，评分值的变化趋于平缓。至少现在聚类使用了所有的输入特征。</p><h3 id="利用标号的熵信息"><a href="#利用标号的熵信息" class="headerlink" title="利用标号的熵信息"></a>利用标号的熵信息</h3><p>标签告诉我们每个数据点的真实性质。好的聚类应该和人工标签保持一致，大部分情况 下，标签相同的数据点应聚在一起，而标签不同的数据点不应该在一起，并且簇内的数据 点标签相同。</p><p>良好的聚类结果簇中样本类别大体相同，因而熵值较低。我们可以对各个簇的熵加权平均，将结果作为聚类得分：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> numpy</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">entropy</span>(<span class="params">x</span>):</span><br><span class="line"> ent = <span class="number">0.0</span></span><br><span class="line"> x_value_list = [x[i] <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(x.shape[<span class="number">0</span>])]</span><br><span class="line"> n=<span class="built_in">sum</span>(x_value_list)</span><br><span class="line"> <span class="keyword">for</span> x_value <span class="keyword">in</span> x_value_list:</span><br><span class="line"> p = <span class="built_in">float</span>(x_value) / n</span><br><span class="line"> ent -= p * numpy.log(p)</span><br><span class="line"> <span class="keyword">return</span> ent</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">fitPipeline4</span>(<span class="params">data, k</span>):</span><br><span class="line"> protoTypeEncoder, protoTypeVecCol = oneHotPipeline(<span class="string">"protocol_type"</span>)</span><br><span class="line"> serviceEncoder, serviceVecCol = oneHotPipeline(<span class="string">"service"</span>)</span><br><span class="line"> flagEncoder, flagVecCol = oneHotPipeline(<span class="string">"flag"</span>)</span><br><span class="line"> assembleCols = (<span class="built_in">set</span>(data.columns)-<span class="built_in">set</span>([<span class="string">"label"</span>, <span class="string">"protocol_type"</span>, <span class="string">"service"</span>, <span class="string">"flag"</span>])).union(<span class="built_in">set</span>([protoTypeVecCol, serviceVecCol, flagVecCol]))</span><br><span class="line"> assembler = VectorAssembler(inputCols=<span class="built_in">list</span>(assembleCols), outputCol=<span class="string">"featureVector"</span>)</span><br><span class="line"> scaler = StandardScaler(inputCol=<span class="string">"featureVector"</span>, outputCol=<span class="string">"scaledFeatureVector"</span>, withStd=<span class="literal">True</span>, withMean=<span class="literal">False</span>)</span><br><span class="line"> kmeans = KMeans().setSeed(<span class="built_in">int</span>(random.random()*<span class="number">10</span>)).setK(k).setPredictionCol(<span class="string">"cluster"</span>).setFeaturesCol(<span class="string">"scaledFeatureVector"</span>).setMaxIter(<span class="number">40</span>).setTol(<span class="number">1.0e-5</span>)</span><br><span class="line"> pipeline = Pipeline().setStages([protoTypeEncoder, serviceEncoder, flagEncoder, assembler, scaler, kmeans])</span><br><span class="line"> pipelineModel = pipeline.fit(data)</span><br><span class="line"> <span class="keyword">return</span> pipelineModel</span><br><span class="line"></span><br><span class="line"><span class="comment"># 良好的聚类结果簇中样本类别大体相同，因而熵值较低。对各个簇的熵加权平均，将结果作为聚类得分</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">clusteringScore4</span>(<span class="params">data, k</span>):</span><br><span class="line"> pipelineModel = fitPipeline4(data, k)</span><br><span class="line"> clusterLabel = pipelineModel.transform(data).select(<span class="string">"cluster"</span>, <span class="string">"label"</span>)</span><br><span class="line"> pd=clusterLabel.toPandas()</span><br><span class="line"> Sum=<span class="number">0</span></span><br><span class="line"> <span class="keyword">for</span> name, group <span class="keyword">in</span> pd.groupby(<span class="string">"cluster"</span>):</span><br><span class="line"> labelsize=group.count()[<span class="number">0</span>]</span><br><span class="line"> a=numpy.array(group.groupby(<span class="string">'label'</span>).count())</span><br><span class="line"> b=[]</span><br><span class="line"> <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="built_in">len</span>(a)):</span><br><span class="line"> <span class="keyword">for</span> j <span class="keyword">in</span> <span class="built_in">range</span>(<span class="built_in">len</span>(a[i])):</span><br><span class="line"> b.append(a[i][j])</span><br><span class="line"> One=labelsize*entropy(numpy.array(b))</span><br><span class="line"> Sum=Sum+One</span><br><span class="line"> <span class="keyword">return</span> Sum/data.count()</span><br><span class="line"></span><br><span class="line"><span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">60</span>, <span class="number">300</span>, <span class="number">30</span>):</span><br><span class="line"> <span class="built_in">print</span>([k, clusteringScore4(dataDF, k)])</span><br></pre></td></tr></tbody></table></figure><p>输出结果：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">[<span class="number">60</span>, <span class="number">0.038993775215004474</span>]</span><br><span class="line">[<span class="number">90</span>, <span class="number">0.02985377476611417</span>]</span><br><span class="line">[<span class="number">120</span>, <span class="number">0.02266161774992263</span>]</span><br><span class="line">[<span class="number">150</span>, <span class="number">0.020766076760220943</span>]</span><br><span class="line">[<span class="number">180</span>, <span class="number">0.017547365257679748</span>]</span><br><span class="line">[<span class="number">210</span>, <span class="number">0.012974819022593053</span>]</span><br><span class="line">[<span class="number">240</span>, <span class="number">0.007150061376894767</span>]</span><br><span class="line">[<span class="number">270</span>, <span class="number">0.00833981903044443</span>]</span><br></pre></td></tr></tbody></table></figure><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651999142438/202082162021.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651999142438/202082162021.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="利用标号的熵信息"></p><p>跟以前一样，可以根据上面的分析结果大致看出 k 的合适取值。随着 k 的增加，熵不一定会减小，因此我们找到的可能是一个局部最小值。这里结果同样表明，k 取 240 可能比较合理，因为它的得分实际上低于 210 以及 270。</p><h3 id="聚类实战"><a href="#聚类实战" class="headerlink" title="聚类实战"></a>聚类实战</h3><p><strong>取 k=180</strong></p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">pipelineModel = fitPipeline4(dataDF, <span class="number">180</span>)</span><br><span class="line">countByClusterLabel = pipelineModel.transform(dataDF).select(<span class="string">"cluster"</span>, <span class="string">"label"</span>).groupBy(<span class="string">"cluster"</span>, <span class="string">"label"</span>).count().orderBy(<span class="string">"cluster"</span>, <span class="string">"label"</span>)</span><br><span class="line">countByClusterLabel.show()</span><br></pre></td></tr></tbody></table></figure><p>这里我们同样把每个簇的标号打印出来。聚类的结果中大部分属于同一簇，以及其他的少部分簇。</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">+-------+----------+-------+ </span><br><span class="line">|cluster| label| count|</span><br><span class="line">+-------+----------+-------+</span><br><span class="line">| <span class="number">0</span>| neptune.| <span class="number">362876</span>|</span><br><span class="line">| <span class="number">0</span>|portsweep.| <span class="number">1</span>|</span><br><span class="line">| <span class="number">1</span>| ipsweep.| <span class="number">40</span>|</span><br><span class="line">| <span class="number">1</span>| nmap.| <span class="number">6</span>|</span><br><span class="line">| <span class="number">1</span>| normal.| <span class="number">3421</span>|</span><br><span class="line">| <span class="number">1</span>|portsweep.| <span class="number">2</span>|</span><br><span class="line">| <span class="number">1</span>| satan.| <span class="number">11</span>|</span><br><span class="line">| <span class="number">1</span>| smurf.|<span class="number">2807886</span>|</span><br><span class="line">| <span class="number">2</span>| neptune.| <span class="number">1038</span>|</span><br><span class="line">| <span class="number">2</span>|portsweep.| <span class="number">13</span>|</span><br><span class="line">| <span class="number">2</span>| satan.| <span class="number">3</span>|</span><br><span class="line">| <span class="number">3</span>| ipsweep.| <span class="number">13</span>|</span><br><span class="line">| <span class="number">3</span>| neptune.| <span class="number">1046</span>|</span><br><span class="line">| <span class="number">3</span>| normal.| <span class="number">38</span>|</span><br><span class="line">| <span class="number">3</span>|portsweep.| <span class="number">11</span>|</span><br><span class="line">| <span class="number">3</span>| satan.| <span class="number">3</span>|</span><br><span class="line">| <span class="number">4</span>| neptune.| <span class="number">1034</span>|</span><br><span class="line">| <span class="number">4</span>| normal.| <span class="number">4</span>|</span><br><span class="line">| <span class="number">4</span>|portsweep.| <span class="number">7</span>|</span><br><span class="line">| <span class="number">4</span>| satan.| <span class="number">4</span>|</span><br><span class="line">+-------+----------+-------+</span><br><span class="line">only showing top <span class="number">20</span> rows</span><br></pre></td></tr></tbody></table></figure><p>现在可以建立一个真正的异常检测系统了。异常检测时需要度量新数据点到最近的簇质心 的距离。如果这个距离超过某个阈值，那么就表示这个新数据点是异常的。我们可以把阈 值设为已知数据中离中心最远的第 100 个点到中心的距离。</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> os, tempfile</span><br><span class="line"><span class="keyword">from</span> pyspark.ml.linalg <span class="keyword">import</span> Vector, Vectors</span><br><span class="line"></span><br><span class="line">pipelineModel = fitPipeline4(dataDF, <span class="number">180</span>)</span><br><span class="line">kmeansModel = pipelineModel.stages[-<span class="number">1</span>]</span><br><span class="line">kmeansModel.save(<span class="string">"/model/3/kmeansModel"</span>)</span><br><span class="line">pipelineModel.save(<span class="string">"/model/3/pipelineModel"</span>)</span><br><span class="line">centroids = kmeansModel.clusterCenters()</span><br><span class="line">clustered = pipelineModel.transform(dataDF)</span><br><span class="line">threshold=clustered.select(<span class="string">"cluster"</span>, <span class="string">"scaledFeatureVector"</span>).rdd.<span class="built_in">map</span>(<span class="keyword">lambda</span> a:Vectors.squared_distance(centroids[a.cluster], a.scaledFeatureVector)).sortBy(<span class="keyword">lambda</span> x: x).take(<span class="number">100</span>)[-<span class="number">1</span>]</span><br><span class="line"><span class="built_in">print</span>(threshold)</span><br></pre></td></tr></tbody></table></figure><p>输出阈值：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="number">3.232811853048799e-05</span></span><br></pre></td></tr></tbody></table></figure><p>最后一步就是在新数据点出现的时候使用阈值进行评估。在 unlabled 数据上进行测试找出异常流量记录，并计算正确率。</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">clustered = pipelineModel.transform(testDF)</span><br><span class="line">anomalies = clustered.rdd.<span class="built_in">filter</span>(<span class="keyword">lambda</span> a:Vectors.squared_distance(centroids[a.cluster], a.scaledFeatureVector) >= threshold).collect()</span><br><span class="line">n=<span class="built_in">len</span>(anomalies)</span><br><span class="line">v=<span class="number">0</span></span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> anomalies:</span><br><span class="line"> <span class="keyword">if</span> i[<span class="string">"label"</span>]!=<span class="string">'normal.'</span>:</span><br><span class="line"> v=v+<span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">"正确率:"</span>+<span class="built_in">str</span>(<span class="built_in">float</span>(v)/n))</span><br></pre></td></tr></tbody></table></figure><p>输出结果：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">正确率:<span class="number">0.8051841158484633</span> </span><br></pre></td></tr></tbody></table></figure><p><strong>取 k=240</strong></p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> os, tempfile</span><br><span class="line"><span class="keyword">from</span> pyspark.ml.linalg <span class="keyword">import</span> Vector, Vectors</span><br><span class="line">pipelineModel = fitPipeline4(dataDF, <span class="number">240</span>)</span><br><span class="line">kmeansModel = pipelineModel.stages[-<span class="number">1</span>]</span><br><span class="line">kmeansModel.save(<span class="string">"/model/3/test/kmeansModel"</span>)</span><br><span class="line">pipelineModel.save(<span class="string">"/model/3/test/pipelineModel"</span>)</span><br><span class="line">centroids = kmeansModel.clusterCenters()</span><br><span class="line">clustered = pipelineModel.transform(dataDF)</span><br><span class="line">threshold=clustered.select(<span class="string">"cluster"</span>, <span class="string">"scaledFeatureVector"</span>).rdd.<span class="built_in">map</span>(<span class="keyword">lambda</span> a:Vectors.squared_distance(centroids[a.cluster], a.scaledFeatureVector)).sortBy(<span class="keyword">lambda</span> x: x).take(<span class="number">100</span>)[-<span class="number">1</span>]</span><br><span class="line"><span class="built_in">print</span>(threshold)</span><br></pre></td></tr></tbody></table></figure><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="number">7.665805787851659e-06</span> </span><br></pre></td></tr></tbody></table></figure><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">clustered = pipelineModel.transform(testDF)</span><br><span class="line">anomalies = clustered.rdd.<span class="built_in">filter</span>(<span class="keyword">lambda</span> a:Vectors.squared_distance(centroids[a.cluster], a.scaledFeatureVector) >= threshold).collect()</span><br><span class="line">n=<span class="built_in">len</span>(anomalies)</span><br><span class="line">v=<span class="number">0</span></span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> anomalies:</span><br><span class="line"> <span class="keyword">if</span> i[<span class="string">"label"</span>]!=<span class="string">'normal.'</span>:</span><br><span class="line"> v=v+<span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">"正确率:"</span>+<span class="built_in">str</span>(<span class="built_in">float</span>(v)/n))</span><br></pre></td></tr></tbody></table></figure><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">正确率:<span class="number">0.8050769488123118</span></span><br></pre></td></tr></tbody></table></figure><p>可以看出 K=180 是在 unlabled 数据上进行测试找出异常流量记录，计算正确率比 K=240 有较好的结果。</p><p>缩短计算的步长：</p><figure class="highlight py"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">for</span> k <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">150</span>, <span class="number">220</span>, <span class="number">10</span>):</span><br><span class="line"> <span class="built_in">print</span>([k, clusteringScore3(dataDF, k)])</span><br></pre></td></tr></tbody></table></figure><p>得到评分结果：</p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">[<span class="number">150</span>, <span class="number">2.292921780026778</span>]</span><br><span class="line">[<span class="number">160</span>, <span class="number">4.917845778754763</span>]</span><br><span class="line">[<span class="number">170</span>, <span class="number">2.0016455721528015</span>]</span><br><span class="line">[<span class="number">180</span>, <span class="number">1.7177635513092788</span>]</span><br><span class="line">[<span class="number">190</span>, <span class="number">1.5766159846344556</span>]</span><br><span class="line">[<span class="number">200</span>, <span class="number">1.5550587983858675</span>]</span><br><span class="line">[<span class="number">210</span>, <span class="number">1.2785418817225693</span>]</span><br></pre></td></tr></tbody></table></figure><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651999198821/202082161957.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651999198821/202082161957.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="评分结果"></p><p>局部放大：</p><p><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651999229193/20208216196.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651999229193/20208216196.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="局部放大"></p><p><strong>取 k=190</strong></p><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="keyword">import</span> os, tempfile</span><br><span class="line"><span class="keyword">from</span> pyspark.ml.linalg <span class="keyword">import</span> Vector, Vectors</span><br><span class="line">pipelineModel = fitPipeline4(dataDF, <span class="number">190</span>)</span><br><span class="line">kmeansModel = pipelineModel.stages[-<span class="number">1</span>]</span><br><span class="line">kmeansModel.save(<span class="string">"/model/3/190/kmeansModel"</span>)</span><br><span class="line">pipelineModel.save(<span class="string">"/model/3/190/pipelineModel"</span>)</span><br><span class="line">centroids = kmeansModel.clusterCenters()</span><br><span class="line">clustered = pipelineModel.transform(dataDF)</span><br><span class="line">threshold=clustered.select(<span class="string">"cluster"</span>, <span class="string">"scaledFeatureVector"</span>).rdd.<span class="built_in">map</span>(<span class="keyword">lambda</span> a:Vectors.squared_distance(centroids[a.cluster], a.scaledFeatureVector)).sortBy(<span class="keyword">lambda</span> x: x).take(<span class="number">100</span>)[-<span class="number">1</span>]</span><br><span class="line"><span class="built_in">print</span>(threshold)</span><br></pre></td></tr></tbody></table></figure><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line"><span class="number">3.247829147436459e-05</span> </span><br></pre></td></tr></tbody></table></figure><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">clustered = pipelineModel.transform(testDF)</span><br><span class="line">anomalies = clustered.rdd.<span class="built_in">filter</span>(<span class="keyword">lambda</span> a:Vectors.squared_distance(centroids[a.cluster], a.scaledFeatureVector) >= threshold).collect()</span><br><span class="line">n=<span class="built_in">len</span>(anomalies)</span><br><span class="line">v=<span class="number">0</span></span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> anomalies:</span><br><span class="line"> <span class="keyword">if</span> i[<span class="string">"label"</span>]!=<span class="string">'normal.'</span>:</span><br><span class="line"> v=v+<span class="number">1</span></span><br><span class="line"></span><br><span class="line"><span class="built_in">print</span>(<span class="string">"正确率:"</span>+<span class="built_in">str</span>(<span class="built_in">float</span>(v)/n))</span><br></pre></td></tr></tbody></table></figure><figure class="highlight python"><table><tbody><tr><td class="code"><pre><span class="line">正确率:<span class="number">0.8051841158484633</span> </span><br></pre></td></tr></tbody></table></figure><p>可以看出 K=190 是在 unlabled 数据上进行测试找出异常流量记录，计算正确率比 K=180 有较好的结果。</p></div></div><div class="footer"></div><div class="article-meta" id="bottom"><div class="new-meta-box"></div></div><div class="prev-next"><a class="prev" href="/notes/Spark/als"><p class="title"><i class="fa-solid fa-chevron-left" aria-hidden="true"></i>基于Audioscrobbler数据集的音乐推荐(pyspark)</p><p class="content"> ALS 基于 Audioscrobbler 数据集的音乐推荐 (pyspark) 根据用户播放次数数据使用协同过滤算法完成音乐推荐。 数据集Audioscrobbler 数据集 下载 ...</p></a></div><div class="recommended-article"><div class="recommended-article-header"><i class="fa-solid fa-bookmark fa-fw" aria-hidden="true"></i> <span>推荐阅读</span></div><div class="recommended-article-group"> <a class="recommended-article-item" href="/notes/Spark/als.html" title="基于Audioscrobbler数据集的音乐推荐(pyspark)" rel="bookmark"><img src="https://static.mhuig.top/npm/imbox@0.0.15/c/66.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.15/c/66.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="基于Audioscrobbler数据集的音乐推荐(pyspark)"> <span class="title">基于Audioscrobbler数据集的音乐推荐(pyspark)</span></a> <a class="recommended-article-item" href="/notes/Spark/rdd.html" title="Spark RDD 编程" rel="bookmark"><img src="https://static.mhuig.top/npm/imbox@0.0.15/c/45.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.15/c/45.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="Spark RDD 编程"> <span class="title">Spark RDD 编程</span></a> <a class="recommended-article-item" href="/p/48f700eb/" title="BERT 预训练模型及其应用案例" rel="bookmark"><img src="https://static.mhuig.top/npm/mhgoos@0.0.1651974150318/2020116102430.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/mhgoos@0.0.1651974150318/2020116102430.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="BERT 预训练模型及其应用案例"> <span class="title">BERT 预训练模型及其应用案例</span></a> <a class="recommended-article-item" href="/p/f0765214/" title="特征向量和特征值的几何本质" rel="bookmark"><img src="https://static.mhuig.top/npm/imbox@0.0.15/c/111.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.15/c/111.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="特征向量和特征值的几何本质"> <span class="title">特征向量和特征值的几何本质</span></a> <a class="recommended-article-item" href="/p/483290b5/" title="特征和分类器" rel="bookmark"><img src="https://static.mhuig.top/npm/imbox@0.0.15/c/4.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.15/c/4.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="特征和分类器"> <span class="title">特征和分类器</span></a> <a class="recommended-article-item" href="/notes/Cryptography/RailFenceCipher.html" title="Python RailFenceCipher" rel="bookmark"><img src="https://static.mhuig.top/npm/imbox@0.0.15/c/70.webp" class="lazyload" data-srcset="https://static.mhuig.top/npm/imbox@0.0.15/c/70.webp" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" alt="Python RailFenceCipher"> <span class="title">Python RailFenceCipher</span></a></div></div></article><article class="post white-box shadow floatable blur" id="comments"><span hidden><meta itemprop="discussionUrl" content="/notes/Spark/k-means#comments"></span><p ct><i class="fa-duotone fa-comments"></i> 留言区</p><div id="layoutHelper-comments"></div></article></div><aside id="l_side" itemscope itemtype="http://schema.org/WPSideBar"><section class="widget text desktop mobile pjax"><header><a href="/notes/"><i class="fa-duotone fa-book fa-fw" aria-hidden="true"></i> <span class="name">Notes</span></a></header><div class="content"></div></section><section class="widget list group desktop mobile pjax"><header><i class="fa-duotone fa-comet fa-fw" aria-hidden="true"></i> <span class="name">Spark</span></header><div class="content"><ul class="list entry navigation"><li><a class="flat-box" title="/notes/Spark/" href="/notes/Spark/" active-action="action-notesSpark"><div class="name"> Welcome</div></a></li><li><a class="flat-box" title="/notes/Spark/env" href="/notes/Spark/env" active-action="action-notesSparkenv"><div class="name"> Environment Deployment</div></a></li><li><a class="flat-box" title="/notes/Spark/rdd" href="/notes/Spark/rdd" active-action="action-notesSparkrdd"><div class="name"> RDD</div></a></li><li><a class="flat-box" title="/notes/Spark/streaming" href="/notes/Spark/streaming" active-action="action-notesSparkstreaming"><div class="name"> Streaming</div></a></li><li><a class="flat-box" title="/notes/Spark/streaming-kafka" href="/notes/Spark/streaming-kafka" active-action="action-notesSparkstreaming-kafka"><div class="name"> Kafka</div></a></li><li><a class="flat-box" title="/notes/Spark/als" href="/notes/Spark/als" active-action="action-notesSparkals"><div class="name"> ALS</div></a></li><li><a class="flat-box" title="/notes/Spark/k-means" href="/notes/Spark/k-means" active-action="action-notesSparkk-means"><div class="name"> K-Means</div></a></li></ul></div></section><div class="widget-sticky pjax"><section class="widget toc-wrapper desktop mobile" id="toc-div"><header><i class="fa-duotone fa-list fa-fw" aria-hidden="true"></i> <span class="name">本文目录</span></header><div class="content"><ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E9%9B%86"><span class="toc-text">数据集</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E7%AE%97%E6%B3%95"><span class="toc-text">算法</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#K%E5%9D%87%E5%80%BC%E8%81%9A%E7%B1%BB%E7%AE%97%E6%B3%95"><span class="toc-text">K 均值聚类算法</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%AE%97%E6%B3%95%E8%BF%87%E7%A8%8B"><span class="toc-text">算法过程</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BC%AA%E4%BB%A3%E7%A0%81"><span class="toc-text">伪代码</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#K-means%E8%81%9A%E7%B1%BB%E6%9C%80%E4%BC%98k%E5%80%BC%E7%9A%84%E9%80%89%E5%8F%96%EF%BC%88%E6%89%8B%E8%82%98%E6%B3%95%EF%BC%89"><span class="toc-text">K-means 聚类最优 k 值的选取（手肘法）</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%89%B9%E5%BE%81%E7%9A%84%E8%A7%84%E8%8C%83%E5%8C%96"><span class="toc-text">特征的规范化</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%B1%BB%E5%88%AB%E5%9E%8B%E5%8F%98%E9%87%8F"><span class="toc-text">类别型变量</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%81%9A%E7%B1%BB%E7%BB%93%E6%9E%9C%E8%AF%84%E4%BB%B7%E6%8C%87%E6%A0%87"><span class="toc-text">聚类结果评价指标</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#Entropy%EF%BC%88%E7%86%B5%EF%BC%89"><span class="toc-text">Entropy（熵）</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#Accuracy-%E5%87%86%E7%A1%AE%E7%8E%87"><span class="toc-text">Accuracy (准确率)</span></a></li></ol></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%AE%9E%E9%AA%8C%E8%BF%87%E7%A8%8B"><span class="toc-text">实验过程</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%87%86%E5%A4%87%E6%95%B0%E6%8D%AE%EF%BC%8C%E4%B8%8A%E4%BC%A0%E8%87%B3HDFS"><span class="toc-text">准备数据，上传至 HDFS</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E9%80%9A%E8%BF%87kddcup-names%E5%8A%A0%E8%BD%BD%E5%88%97%E5%90%8D%E7%A7%B0"><span class="toc-text">通过 kddcup.names 加载列名称</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%9E%84%E5%BB%BADataframe"><span class="toc-text">构建 Dataframe</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E9%9B%86%E7%BB%9F%E8%AE%A1"><span class="toc-text">数据集统计</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%B0%9D%E8%AF%95%E8%81%9A%E7%B1%BB"><span class="toc-text">尝试聚类</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#k%E7%9A%84%E9%80%89%E6%8B%A9"><span class="toc-text">k 的选择</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%89%B9%E5%BE%81%E7%9A%84%E8%A7%84%E8%8C%83%E5%8C%96-1"><span class="toc-text">特征的规范化</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%B1%BB%E5%88%AB%E5%9E%8B%E5%8F%98%E9%87%8F-1"><span class="toc-text">类别型变量</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%A9%E7%94%A8%E6%A0%87%E5%8F%B7%E7%9A%84%E7%86%B5%E4%BF%A1%E6%81%AF"><span class="toc-text">利用标号的熵信息</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%81%9A%E7%B1%BB%E5%AE%9E%E6%88%98"><span class="toc-text">聚类实战</span></a></li></ol></li></ol></div></section></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div><div class="pjax"></div></aside><pjax><script>window.pdata={},pdata.ispage=!1,pdata.commentPath="",pdata.commentPlaceholder="",pdata.commentConfig={};var l_header=document.getElementById("l_header");l_header.classList.add("show");var cover_wrapper=document.querySelector("#l_cover .cover-wrapper"),scroll_down=document.getElementById("scroll-down");cover_wrapper.id="none",cover_wrapper.style.display="none",scroll_down.style.display="none"</script></pjax></div><footer class="footer clearfix" itemscope itemtype="http://schema.org/WPFooter"><br><br><div class="aplayer-container"><div class="aplayer-local"></div></div><br><div class="social-wrapper" itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/atom.xml" class="social fa-duotone fa-rss flat-btn" target="_blank" rel="external nofollow noopener noreferrer" itemprop="url"></a><a href="https://mhuig.top/contact/" class="social fa-duotone fa-envelope flat-btn" target="_blank" rel="external nofollow noopener noreferrer" itemprop="url"></a><a href="https://github.com/MHuiG" class="social fa-brands fa-github flat-btn" target="_blank" rel="external nofollow noopener noreferrer" itemprop="url"></a><a href="https://t.me/MHuiG" class="social fa-brands fa-telegram flat-btn" target="_blank" rel="external nofollow noopener noreferrer" itemprop="url"></a></div><div><p>博客内容遵循 <a target="_blank" rel="external nofollow noopener noreferrer" href="https://creativecommons.org/licenses/by-nc-sa/4.0/deed.zh">署名-非商业性使用-相同方式共享 4.0 国际 (CC BY-NC-SA 4.0) 协议</a></p></div><div class="copyright"><p><a href="https://icp.gov.moe" target="_blank" rel="external nofollow noopener noreferrer">萌ICP备</a> <a href="https://icp.gov.moe/?keyword=2020012138" target="_blank" rel="external nofollow noopener noreferrer">2020012138号</a><br><a target="_blank" rel="noopener" href="https://mhuig.top/">Copyright © Since 2018 MHuiG</a></p></div></footer><a id="s-top" class="fa-solid fa-arrow-up fa-fw" title="top"></a></div></div><div><script>volantis.dom.bodyAnchor=volantis.dom.$(document.getElementById("safearea")),volantis.dom.topBtn=volantis.dom.$(document.getElementById("s-top")),volantis.dom.wrapper=volantis.dom.$(document.getElementById("wrapper")),volantis.dom.coverAnchor=volantis.dom.$(document.querySelector("#l_cover .cover-wrapper")),volantis.dom.switcher=volantis.dom.$(document.querySelector("#l_header .switcher .s-search")),volantis.dom.header=volantis.dom.$(document.getElementById("l_header")),volantis.dom.search=volantis.dom.$(document.querySelector("#l_header .m_search")),volantis.dom.mPhoneList=volantis.dom.$(document.querySelectorAll("#l_header .m-phone .list-v"))</script><script>volantis.css("https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/libs/@fortawesome/fontawesome-pro/css/all.min.css")</script><script src="/js/app.js?v=196690b6a0"></script><script>const rootElement=document.documentElement,darkModeStorageKey="color-scheme",rootElementDarkModeAttributeName="color-scheme",setLS=(e,t)=>{localStorage.setItem(e,t)},removeLS=e=>{localStorage.removeItem(e)},getLS=e=>localStorage.getItem(e),getModeFromCSSMediaQuery=()=>window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light",resetRootDarkModeAttributeAndLS=()=>{rootElement.removeAttribute("color-scheme"),removeLS("color-scheme")},validColorModeKeys={dark:!0,light:!0},applyCustomDarkModeSettings=e=>{const t=e||getLS("color-scheme");getCustomDarkMode(),t===getModeFromCSSMediaQuery()?resetRootDarkModeAttributeAndLS():validColorModeKeys[t]?rootElement.setAttribute("color-scheme",t):resetRootDarkModeAttributeAndLS()},invertDarkModeObj={dark:"light",light:"dark"},getCustomDarkMode=()=>{let e=getLS("color-scheme");if(validColorModeKeys[e])e=invertDarkModeObj[e];else{if(null!==e)return;e=invertDarkModeObj[getModeFromCSSMediaQuery()]}volantis.dark.mode="dark"==e?"light":"dark"},toggleCustomDarkMode=()=>{let e=getLS("color-scheme");if(validColorModeKeys[e])e=invertDarkModeObj[e];else{if(null!==e)return;e=invertDarkModeObj[getModeFromCSSMediaQuery()]}return setLS("color-scheme",e),e};function bindToggleButton(){document.querySelectorAll("#wrapper .toggle-mode-btn,#rightmenu-wrapper .toggle-mode-btn").forEach((function(e){volantis.dom.$(e).on("click",volantis.dark.toggle)}))}volantis.dark.toggle=()=>{const e=toggleCustomDarkMode();applyCustomDarkModeSettings(e),volantis.dark.method.toggle.start()},applyCustomDarkModeSettings(),document.addEventListener("DOMContentLoaded",(()=>{volantis.requestAnimationFrame(bindToggleButton)})),volantis.pjax.push(bindToggleButton);const darkModelListeners={dark:e=>{e.matches&&(volantis.dark.mode="dark"),volantis.dark.method.toggle.start()},light:e=>{e.matches&&(volantis.dark.mode="light"),volantis.dark.method.toggle.start()}};window.matchMedia("(prefers-color-scheme: dark)").addListener(darkModelListeners.dark),window.matchMedia("(prefers-color-scheme: light)").addListener(darkModelListeners.light)</script><script>function loadIssuesJS(){null!=document.getElementById("sites-api")&&"undefined"==typeof SitesJS&&volantis.js("/js/plugins/tags/sites.js?v=a7e55dacdf");null!=document.getElementById("friends-api")&&"undefined"==typeof FriendsJS&&volantis.js("/js/plugins/tags/friends.js?v=187f37f894");null!=document.getElementById("contributors-api")&&"undefined"==typeof ContributorsJS&&volantis.js("/js/plugins/tags/contributors.js?v=16e248333a")}loadIssuesJS(),volantis.pjax.push((()=>{loadIssuesJS()}))</script><script defer="defer" src="https://cdn.bootcdn.net/ajax/libs/vanilla-lazyload/17.1.0/lazyload.min.js"></script><script>window.lazyLoadOptions={elements_selector:".lazyload",threshold:0},window.addEventListener("LazyLoad::Initialized",(function(n){window.lazyLoadInstance=n.detail.instance}),!1),document.addEventListener("DOMContentLoaded",(function(){lazyLoadInstance.update()})),document.addEventListener("pjax:complete",(function(){lazyLoadInstance.update()}))</script><script>window.FPConfig={delay:0,ignoreKeywords:["#"],maxRPS:6,hoverDelay:0}</script><script defer="defer" src="https://static.mhuig.top/npm/volantis-static@0.0.1660614606622/libs/flying-pages/flying-pages.min.js"></script><script type="text/javascript">function pjax_scrollrebeal(){ScrollReveal().reveal("#l_main .reveal",{distance:"32px",duration:"800",interval:"20",scale:"1",easing:"ease-out"})}function init_scrollrebeal(){"undefined"==typeof ScrollReveal?volantis.requestAnimationFrame(init_scrollrebeal):pjax_scrollrebeal()}volantis.js("https://cdn.bootcdn.net/ajax/libs/scrollReveal.js/4.0.9/scrollreveal.min.js"),document.addEventListener("DOMContentLoaded",init_scrollrebeal),volantis.pjax.push(pjax_scrollrebeal,"pjax_scrollrebeal",setRequestAnimationFrame=!1)</script><script>volantis.css("https://cdn.bootcdn.net/ajax/libs/aplayer/1.10.1/APlayer.min.css"),volantis.js("https://cdn.bootcdn.net/ajax/libs/aplayer/1.10.1/APlayer.min.js").then((()=>{document.querySelectorAll(".aplayer-local").forEach((e=>{new APlayer({container:e,fixed:!1,order:"list",volume:"0.7",autoplay:!1,loop:"all",theme:"#1BCDFC",listMaxHeight:"320px",listFolded:"true",preload:"auto",lrcType:3,audio:[{name:"虹之间",artist:"钢琴版伴奏",url:"https://blog.mhuig.top/music-Archive/BetweenTheRainbow.mp3",lrc:"https://blog.mhuig.top/music-Archive/BetweenTheRainbow.lrc",cover:"https://blog.mhuig.top/music-Archive/BetweenTheRainbow.jpg"},{name:"Avem",artist:"Alan Walker",url:"https://blog.mhuig.top/music-Archive/Avem.mp3",lrc:"https://blog.mhuig.top/music-Archive/Avem.lrc",cover:"https://blog.mhuig.top/music-Archive/Avem.jpg"},{name:"Fly",artist:"Marshmello / Leau Culver",url:"https://blog.mhuig.top/music-Archive/fly.mp3",lrc:"https://blog.mhuig.top/music-Archive/fly.lrc",cover:"https://blog.mhuig.top/music-Archive/fly.jpg"},{name:"Alone",artist:"Marshmello",url:"https://blog.mhuig.top/music-Archive/alone.mp3",lrc:"https://blog.mhuig.top/music-Archive/alone.lrc",cover:"https://blog.mhuig.top/music-Archive/alone.jpg"}]})}))}))</script><script>function check_giscus(){return"dark"===volantis.dark.mode?volantis.giscus.Theme="dark":volantis.giscus.Theme="light",document.getElementById("giscus_container")}function pjax_giscus(){const t=check_giscus();if(!t)return;let s=Object.assign({theme:{light:"light",dark:"dark"},repo:"MHuiG/talk","repo-id":"MDEwOlJlcG9zaXRvcnkyODQ4NDg1NTg=",category:"Giscus","category-id":"DIC_kwDOEPpxrs4B_X67",mapping:"pathname","reactions-enabled":"1","emit-metadata":"1",lang:"zh-CN","input-position":"top"},pdata.commentConfig);const e=document.createElement("script");e.setAttribute("src","https://giscus.app/client.js"),Object.keys(s).forEach((t=>{"theme"!=t&&e.setAttribute("data-"+t,s[t])})),e.setAttribute("data-theme",volantis.giscus.Theme),e.setAttribute("crossorigin","anonymous"),t.appendChild(e)}function dark_giscus(){if(!check_giscus())return;const t={setConfig:{theme:volantis.giscus.Theme}};document.querySelector("iframe.giscus-frame").contentWindow.postMessage({giscus:t},"https://giscus.app")}volantis.layoutHelper("comments",'<div id="giscus_container"></div>'),volantis.giscus={},pjax_giscus(),volantis.pjax.push(pjax_giscus),volantis.dark.push(dark_giscus)</script><script>async function loadSearchScript(){return volantis.js("/js/search/hexo.js?v=e2a1e6a8d9");}function loadSearchService(){loadSearchScript(),document.querySelectorAll(".input.u-search-input").forEach((e=>{e.removeEventListener("focus",loadSearchService,!1)})),document.querySelectorAll(".u-search-form").forEach((e=>{e.addEventListener("submit",(e=>{e.preventDefault()}),!1)}))}function OpenSearch(e){"undefined"==typeof SearchService?loadSearchScript().then((()=>{SearchService.setQueryText(e),SearchService.search()})):(SearchService.setQueryText(e),SearchService.search())}if(window.location.search&&/^\?s=/g.test(window.location.search)){OpenSearch(decodeURI(window.location.search).replace(/\ /g,"-").replace(/^\?s=/g,""))}document.querySelectorAll(".input.u-search-input").forEach((e=>{e.addEventListener("focus",loadSearchService,!1)}))</script><script>function pjax_highlightjs_copyCode(){(document.querySelector(".highlight .code pre")||document.querySelector(".article pre code"))&&VolantisApp.utilCopyCode(".highlight .code pre, .article pre code")}volantis.requestAnimationFrame(pjax_highlightjs_copyCode),volantis.pjax.push(pjax_highlightjs_copyCode)</script><script>function load_swiper(){document.querySelectorAll(".swiper-container")[0]&&(volantis.css("https://cdn.bootcdn.net/ajax/libs/Swiper/8.3.2/swiper-bundle.min.css"),volantis.js("https://cdn.bootcdn.net/ajax/libs/Swiper/8.3.2/swiper-bundle.min.js").then((()=>{pjax_swiper()})))}function pjax_swiper(){volantis.swiper=new Swiper(".swiper-container",{slidesPerView:"auto",spaceBetween:8,centeredSlides:!0,loop:!0,pagination:{el:".swiper-pagination",clickable:!0},navigation:{nextEl:".swiper-button-next",prevEl:".swiper-button-prev"}})}load_swiper(),volantis.pjax.push((()=>{document.querySelectorAll(".swiper-container")[0]&&(void 0===volantis.swiper?load_swiper():pjax_swiper())}))</script><script>volantis.css("https://cdn.bootcdn.net/ajax/libs/pace/1.2.4/themes/blue/pace-theme-minimal.min.css"),volantis.js("https://cdn.bootcdn.net/ajax/libs/pace/1.2.4/pace.min.js").then((()=>{Pace.options.restartOnPushState=!1,volantis.pjax.send(Pace.restart),volantis.scroll.push((()=>{Pace.stop(),Pace.trigger("start"),Pace.bar.update(100*volantis.scroll.progress()),document.body.className=document.body.className.replaceAll("pace-running ","")}),"阅读进度")}))</script><pjax></pjax><script>function listenSidebarTOC(){const t=document.querySelectorAll(".toc li");if(!t.length)return;const e=[];Array.from(t).forEach((t=>{const n=t.querySelector(".toc-link"),i=document.getElementById(n.getAttribute("href")?decodeURI(n.getAttribute("href")).replace("#",""):n.getAttribute("toc-action").split("toc-")[1]);return e.push(i),n.getAttribute("href")&&(n.setAttribute("toc-action","toc-"+decodeURI(n.getAttribute("href")).replace("#","")),n.removeAttribute("href")),i&&i.id&&n.addEventListener("click",(t=>{t.preventDefault(),volantis.scroll.to(i,{addTop:5,observer:!0}),history.pushState(null,document.title,"#"+i.id)})),i}));function n(t){if(t.classList.contains("active-current"))return;document.querySelectorAll(".toc .active").forEach((t=>{t.classList.remove("active","active-current")})),t.classList.add("active","active-current");let e=t.parentNode;for(;!e.matches(".toc");)e.matches("li")&&e.classList.add("active"),e=e.parentNode}volantis.activateNavIndex=0,n(t[volantis.activateNavIndex]),e[0]&&volantis.scroll.push((()=>{if(e[0].getBoundingClientRect().top>=0)volantis.activateNavIndex=0;else if(e[e.length-1].getBoundingClientRect().top<0)volantis.activateNavIndex=e.length-1;else for(let t=0;t<e.length;t++){const n=e[t],i=e[(t+1)%e.length];if(n.getBoundingClientRect().top<0&&i.getBoundingClientRect().top>=0){volantis.activateNavIndex=t;break}}n(t[volantis.activateNavIndex])}))}document.addEventListener("DOMContentLoaded",(()=>{volantis.requestAnimationFrame(listenSidebarTOC)})),volantis.pjax.push(listenSidebarTOC)</script><script>try{let e=(e,t,n=2)=>Math.abs(e-t)<=n,t=(t,n)=>!e(t.width,n.width)||!e(t.height,n.height),n=new WeakMap,i=(e,i=e.getClientBoundingRect())=>{let o=n.get(e);o&&!t(o,i)||(n.set(e,i),e.style["contain-intrinsic-size"]=`${i.width}px ${i.height}px`)},o=new IntersectionObserver(((e,t)=>{e.forEach((e=>{i(e.target,e.boundingClientRect)}))}),{rootMargin:"500px 0px 500px 0px"}),r=new ResizeObserver(((e,t)=>{e.forEach((e=>{i(e.target,e.contentRect)}))})),s=e=>{let t=document.querySelectorAll(e);t.length&&(t.forEach((e=>{o.observe(e),r.observe(e)})),requestAnimationFrame((()=>{requestAnimationFrame((()=>{t[0].style["content-visibility"]="auto"}))})))},a=()=>{"content-visibility"in document.documentElement.style&&s(".post-story")};a(),volantis.pjax.push(a)}catch(e){console.log(e)}</script><script>document.onreadystatechange=function(){if("complete"==document.readyState){const{saveData:e,effectiveType:t}=navigator.connection||navigator.mozConnection||navigator.webkitConnection||{};("none"==getComputedStyle(document.querySelector("#safearea"),null).display||e||/2g/.test(t))&&(document.querySelectorAll(".reveal").forEach((function(e){e.style.opacity="1"})),document.querySelector("#safearea").style.display="block")}}</script><script type="application/ld+json">[{"@context":"http://schema.org","@type":"Organization","name":"MHuiG's Blog","url":"https://blog.mhuig.top/","logo":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3","width":192,"height":192}},{"@context":"http://schema.org","@type":"Person","name":"MHuiG","image":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3"},"url":"https://blog.mhuig.top/","sameAs":["https://github.com/MHuiG","https://twitter.com/iMHuiG","https://t.me/MHuiG","https://keybase.io/MHuiG"],"description":"「Be Yourself, Make a Difference.」"},{"@context":"http://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"item":{"@id":"https://blog.mhuig.top/","name":"MHuiG"}},{"@type":"ListItem","position":2,"item":{"@id":"https://blog.mhuig.top/categories/大数据/","name":"大数据"}},{"@type":"ListItem","position":2,"item":{"@id":"https://blog.mhuig.top/categories/大数据/Spark/","name":"Spark"}},{"@type":"ListItem","position":3,"item":{"@id":"https://blog.mhuig.top/notes/Spark/k-means.html","name":"基于K均值聚类的网络流量异常检测(pyspark)"}}]},{"@context":"http://schema.org","@type":"WebSite","name":"MHuiG's Blog","url":"https://blog.mhuig.top/","keywords":"Blog, IT, BigData, MHuiG, @MHuiG, iMHuiG, 博客, 梦幻岛, Neverland, 技术博客","description":"MHuiG&#39;s Blog (MHuiG的博客) MHuiG&#39;s Neverland（MHuiG的梦幻岛） —— MHuiG(@MHuiG) 随便写写画画的地方 - 技术博客","author":{"@type":"Person","name":"MHuiG","image":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3"},"url":"https://blog.mhuig.top/","description":"「Be Yourself, Make a Difference.」"},"publisher":{"@type":"Organization","name":"MHuiG's Blog","url":"https://blog.mhuig.top/","logo":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3","width":192,"height":192}},"potentialAction":{"@type":"SearchAction","name":"Coco | The Cat of MHuiG","target":{"@type":"EntryPoint","urlTemplate":"https://blog.mhuig.top/havefun/Coco/?s={search_term_string}"},"query-input":"required name=search_term_string"}},{"@context":"http://schema.org","@type":"BlogPosting","headline":"基于K均值聚类的网络流量异常检测(pyspark)","description":"基于K均值聚类的网络流量异常检测(pyspark) 异常检测常用于检测欺诈、网络攻击、服务器及传感设备故障。在这些应用中，我们要能够找出以前从未见过的新型异常，如新欺诈方式、新入侵方法或新服务器故障模式。","inLanguage":"zh-CN","mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.mhuig.top/notes/Spark/k-means"},"author":{"@type":"Person","name":"MHuiG","image":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3"},"url":"https://blog.mhuig.top/"},"publisher":{"@type":"Organization","name":"MHuiG's Blog","logo":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3","width":192,"height":192}},"url":"https://blog.mhuig.top/notes/Spark/k-means","wordCount":0,"datePublished":"2020-08-02T10:23:00.000Z","dateModified":"2022-05-10T02:11:00.000Z","articleSection":"大数据","keywords":"Spark","image":{"@type":"ImageObject","url":"/lib/favicon/android-chrome-192x192.png?v=8bf02188e3","width":192,"height":192}}]</script><script src="https://cdn.bootcdn.net/ajax/libs/pjax/0.2.8/pjax.min.js"></script><script>var pjax;document.addEventListener("DOMContentLoaded",(function(){pjax=new Pjax({elements:'a[href]:not([href^="#"]):not([href="javascript:void(0)"]):not([pjax-fancybox]):not([onclick="return false;"]):not([onclick="return!1"]):not([target="_blank"]):not([target="view_window"]):not([href$=".xml"])',selectors:["head title","head meta[name=keywords]","head meta[name=description]","#l_main","#pjax-header-nav-list",".pjax","pjax"],cacheBust:!1,timeout:5e3})})),document.addEventListener("pjax:send",(function(e){try{var t=window.location.pathname,n=e.triggerElement.href,a=[""];""!=a[0]&&a.forEach((e=>{-1==t.indexOf(e)&&-1==n.indexOf(e)||(window.location.href=n)}))}catch(e){}volantis.pjax.method.send.start()})),document.addEventListener("pjax:complete",(function(){document.querySelectorAll("script[data-pjax], .pjax-reload script").forEach((e=>{const t=e.text||e.textContent||e.innerHTML||"",n=document.createElement("script");Object.keys(e.attributes).forEach((t=>{n.setAttribute(e.attributes[t].nodeName,e.attributes[t].nodeValue)})),t&&n.appendChild(document.createTextNode(t)),e.parentNode.replaceChild(n,e)})),volantis.pjax.method.complete.start()})),document.addEventListener("pjax:error",(function(e){"pjax"===volantis.debug?(console.error(e),console.log("pjax error: \n"+JSON.stringify(e))):(volantis.pjax.method.error.start(),window.location.href=e.triggerElement.href)}))</script></div><pjax> <a style="display:none" target="_blank" rel="external nofollow noopener noreferrer" href="https://bot-trap.mhuig.top/">Are You A Robot?</a></pjax><script>top.location!=self.location&&(top.location=self.location),"mhuig.github.io"==window.location.host||"localhost:4000"==window.location.host||/mhuig\.top$/.test(window.location.host)||/kix\.moe$/.test(window.location.host)||/mhuig/.test(window.location.host)||/ipfs/.test(window.location.host)||(window.location.href="https://blog.mhuig.top/")</script></body></html>